ai-research-survey

scan.json (25860B)

---

 {
   "paper": {
     "title": "LimAgents: Multi-Agent LLMs for Generating Research Limitations",
     "authors": ["Ibrahim Al Azher", "Zhishuai Guo", "Hamed Alhoori"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2601.11578",
     "doi": "10.48550/arXiv.2601.11578"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided in the abstract: https://github.com/IbrahimAlAzhar/LimAgents."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "HuggingFace dataset link provided in the abstract: https://huggingface.co/datasets/iaadlab/LimAgents."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The code link is given but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables I-VII are reported as point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements (e.g., '+15.51%', '+4.41%') based solely on comparing numbers with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as percentage point improvements over baselines with context, e.g., '+15.51 point improvement over its zero-shot baseline' (from 49.43% to 64.94%), '+4.41 point improvement over the zero-shot baseline (62.04%)' in Section VII."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset comprises 2,700 NeurIPS papers but no justification is given for why this number was chosen or whether it provides sufficient statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experimental results. It is unclear whether experiments were run multiple times."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Zero-shot prompting baselines are included for both Llama 3 8B and GPT-4o mini (Table I)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only baselines are zero-shot prompting with the same models. No comparison against other limitation generation systems (e.g., BAGELS framework [1], LimGen [14], or other multi-agent review systems like AgentReview or MARG) is provided."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablation studies in Section VIII: agent quantity, citation agent context, input granularity, core agent contributions (Tables VI, VII), feedback iterations, and chunk enrichment."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: Ground Truth Coverage, ROUGE-L, BLEU, Cosine Similarity, Jaccard Similarity, plus qualitative metrics (Faithfulness, Soundness, Importance)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation conducted: 500-sample annotation study with 3 graduate students for data extraction verification, and a 100-pair validation study with 2 independent annotators for LLM-as-Judge reliability (Section III and V.A)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No mention of train/dev/test splits. It is unclear whether the evaluation was performed on data separate from any tuning or development decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-agent breakdowns provided in Tables IV, V, VI showing performance of individual agents across multiple quality dimensions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases extensively discussed: Graph Agent caused -50.74 point performance collapse, Gemini 1.5 Flash failed in Extractor role, DeepSeek was incompatible, 9-agent setup was counterproductive, feedback degraded Llama 3 performance (Sections VII-VIII)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results reported: feedback loop reducing coverage by -12.62 points (Table II), 9-agent config -12.23 points worse, Graph Agent -50.74 points, dual-stage RAG unsuccessful, specialized agents consistently underperforming (Sections VII-VIII)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '+15.51% coverage gain' for GPT-4o mini 4-agent and '+4.41% improvement' for Llama 3 8B 3-agent are supported by Table I results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'agent-based design distributes complex reasoning' and attributes improvements to the framework design. However, as an observational comparison (not controlled for confounds like prompt length, number of LLM calls, or token budget), these causal attributions are not adequately justified."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract frame this as a general framework for 'generating research limitations' but experiments are limited to NeurIPS 2022-2023 papers with only two models (Llama 3 8B and GPT-4o mini). The limitations section acknowledges this but the framing throughout is broad."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the improvements. For example, the multi-agent setup uses more tokens/compute than zero-shot — the improvement could be due to increased inference budget rather than agent decomposition. This confound is not addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures 'Ground Truth Coverage' but frames results as showing the framework generates 'more meaningful and contextually grounded limitations.' The ground truth itself is constructed via LLM extraction from author statements and OpenReview, yet this proxy is treated as measuring limitation quality without discussing the gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models referred to as 'Llama 3 8B', 'GPT-4o mini', 'DeepSeek R1 Qwen Distil', 'Gemini' without specific version identifiers, snapshot dates, or API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for all agents (Extractor, Analyzer, Reviewer, Citation, Master, Judge, Evaluation) provided in Figures 2-9 in the Appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only 'zero temperature for consistency' mentioned for extraction and feedback threshold of 8/10. No temperature, top-p, or max tokens settings reported for the worker agents, master agent, or evaluation."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed scaffolding description in Sections IV and Figure 1: sequential agent pipeline, RAG system with hybrid BM25+FAISS retrieval, LLM re-ranker, self-feedback loop, quality thresholds, master agent consolidation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing described in Section III: ScienceParse for text extraction, Selenium for OpenReview scraping, rule-based limitation extraction with keyword scanning and section filtering, LLM refinement, and merger/deduplication steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section XI 'Limitations and Future Work' provides a dedicated limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: findings based solely on NeurIPS dataset, only two models tested, input truncation for Llama, human evaluation confined to extraction task, prompt incompatibility with certain reasoning models (Section XI)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section XI explicitly states boundaries: 'findings are based solely on the NeurIPS dataset and only two models (Llama 3 8B and GPT-4o mini), which restricts generalizability.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace (https://huggingface.co/datasets/iaadlab/LimAgents) enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes data collection: 2,700 NeurIPS papers from 2022-2023, ScienceParse for parsing, Selenium for OpenReview scraping, two-stream extraction (author-stated and reviewer-stated), with detail on the pipeline."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the human evaluation, 3 graduate students with ML/NLP expertise were recruited but no description of how they were selected, potential biases, or recruitment process. The 2 annotators for the validation study are also undescribed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the pipeline is described qualitatively, key numbers are missing. The paper states 2,700 papers yielded 51,300 limitations total but doesn't document how many were filtered at each stage or how many papers failed parsing."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors' affiliations at Northern Illinois University Department of Computer Science are clearly listed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence of funder cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for Llama 3 8B or GPT-4o mini. These models were evaluated on NeurIPS 2022-2023 papers which could be in their training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether NeurIPS 2022-2023 papers (and their OpenReview comments) appeared in the training data of the LLMs used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "NeurIPS 2022-2023 papers and OpenReview comments were publicly available before Llama 3 and GPT-4o mini training. The models may have seen both the papers and the ground truth limitations. This is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study — the human annotators evaluated system outputs rather than being study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study in the traditional sense. Annotators evaluated extraction quality."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Annotators are described only as 'graduate students with ML/NLP expertise.'"
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or wall-clock time reported despite the framework making many LLM calls per paper (multiple agents × 2,700 papers)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend mentioned."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity. Results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results are from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget reported despite multiple design choices (feedback threshold 8/10, RAG top-20, relevance score ≥8, etc.)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Many configurations were tried (Tables I, VII) but no clear validation set or principled selection procedure is described. The 'proposed' configurations appear selected based on test set performance."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons across configurations, models, and metrics with no multiple comparison correction or even statistical tests."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Authors evaluate their own framework against their own zero-shot baselines without acknowledging self-comparison bias or seeking independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The multi-agent framework makes far more LLM calls than zero-shot baselines (4+ agent calls + judge + self-feedback + master vs. 1 call). This compute difference is never discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The ground truth is constructed via LLM extraction from author statements and OpenReview comments, then used to evaluate LLM-generated outputs. No discussion of whether this LLM-constructed ground truth is valid or complete for measuring limitation generation quality."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The RAG pipeline, retrieval system, and agent scaffolding are intertwined with the model comparison. When comparing Llama 3 8B vs GPT-4o mini, the scaffold confound (different optimal configs) is not isolated."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "NeurIPS 2022-2023 papers and their OpenReview comments were publicly available before model training. The LLMs may have memorized the ground truth limitations. This temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The Citation Agent accesses cited/citing papers which may contain limitation-related content overlapping with the ground truth."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential non-independence between papers (e.g., papers from the same research group, papers citing each other, shared limitations)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used."
       }
     }
   },
   "claims": [
     {
       "claim": "RAG + multi-agent GPT-4o mini achieves +15.51% coverage gain over zero-shot baseline",
       "evidence": "Table I: GPT-4o mini 4-Agent achieves 64.94% CGT vs 49.43% zero-shot baseline (Section VII.A)",
       "supported": "strong"
     },
     {
       "claim": "Llama 3 8B 3-agent setup yields +4.41% improvement over zero-shot baseline",
       "evidence": "Table I: Llama 3 8B 3-Agent achieves 66.45% vs 62.04% zero-shot (Section VII.A)",
       "supported": "moderate"
     },
     {
       "claim": "Optimized 3-Agent Llama 3 8B outperforms GPT-4o mini's optimal 4-Agent configuration by +1.51 points",
       "evidence": "Table I: Llama 3 8B 3-Agent 66.45% vs GPT-4o mini 4-Agent 64.94% (Section VII.A)",
       "supported": "weak"
     },
     {
       "claim": "Feedback-driven refinement enhances quality but reduces coverage breadth",
       "evidence": "Table II: Coverage drops from 66.45% to 53.83% with feedback, while LLM-generated text coverage improves from 36.59% to 44.77% (Section VII.B)",
       "supported": "moderate"
     },
     {
       "claim": "Excessive agent decomposition (9 agents) is counterproductive, causing -12.23 point drop",
       "evidence": "Table I: 9-agent config achieves 49.81% vs 62.04% zero-shot (Section VIII)",
       "supported": "moderate"
     },
     {
       "claim": "Human-LLM judge agreement is 0.98 and 0.95 for the two annotators",
       "evidence": "Section V.A: validation on 100 random pairs with two independent annotators",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LimAgents, a multi-agent LLM framework for generating research limitations, outperforms zero-shot prompting for both Llama 3 8B (+4.41pp) and GPT-4o mini (+15.51pp) on ground truth coverage. Optimal agent configuration depends on model capacity: smaller models benefit from focused 3-agent setups while larger models leverage 4-agent configurations including a Citation Agent with RAG. Feedback-driven refinement improves quality at the expense of coverage breadth, and excessive agent decomposition (9 agents) degrades performance.",
   "red_flags": [
     {
       "flag": "Compute confound in comparisons",
       "detail": "Multi-agent configurations make 4-6+ LLM calls per paper vs 1 for zero-shot baselines. The improvement may be attributable to increased inference budget rather than agent decomposition. No compute-matched baselines are included."
     },
     {
       "flag": "No statistical tests for any claims",
       "detail": "All claims of improvement are based on comparing single point estimates with no significance tests, confidence intervals, or variance reporting across the 2,700-paper corpus."
     },
     {
       "flag": "LLM-constructed ground truth evaluated by LLM",
       "detail": "Ground truth limitations are extracted by GPT-4o mini, and the pointwise evaluation judge is also GPT-4o mini. This circular dependency could inflate coverage scores if the judge and generator share similar biases."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "NeurIPS 2022-2023 papers and OpenReview comments were public before model training. LLMs may have memorized limitations, inflating both ground truth extraction quality and generation coverage."
     },
     {
       "flag": "Cross-model comparison not controlled",
       "detail": "The claim that Llama 3 8B 3-agent outperforms GPT-4o mini 4-agent (+1.51 points) compares different input configurations and model capacities without controlling for confounds."
     }
   ],
   "cited_papers": [
     {
       "title": "BAGELS: Benchmarking the Automated Generation and Extraction of Limitations from Scholarly Text",
       "authors": ["Ibrahim Al Azher"],
       "year": 2025,
       "arxiv_id": "2505.18207",
       "relevance": "Benchmark framework for limitation generation/extraction that this paper builds upon."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei"],
       "year": 2022,
       "relevance": "Foundational prompting technique that inspired the 'chain of limitations' approach used in LimAgents."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Directly relevant to understanding failure modes of multi-agent LLM systems."
     },
     {
       "title": "AgentReview: Exploring Peer Review Dynamics with LLM Agents",
       "authors": ["Yiqiao Jin"],
       "year": 2024,
       "arxiv_id": "2406.12708",
       "relevance": "Multi-agent LLM system for peer review that models reviewers, authors, and area chairs."
     },
     {
       "title": "Are we there yet? Revealing the risks of utilizing large language models in scholarly peer review",
       "authors": ["Rui Ye"],
       "year": 2024,
       "arxiv_id": "2412.01708",
       "relevance": "Evaluates risks and shortcomings of LLM-based peer review including hallucination and bias."
     },
     {
       "title": "Can large language models provide useful feedback on research papers? A large-scale empirical analysis",
       "authors": ["Weixin Liang"],
       "year": 2024,
       "relevance": "Large-scale study of LLM feedback quality on research papers."
     },
     {
       "title": "Is LLM a reliable reviewer? A comprehensive evaluation of LLM on automatic paper reviewing tasks",
       "authors": ["Ruiyang Zhou"],
       "year": 2024,
       "relevance": "Evaluation of LLM reliability for automatic paper reviewing."
     },
     {
       "title": "OpenReviewer: A specialized large language model for generating critical scientific paper reviews",
       "authors": ["Maximilian Idahl"],
       "year": 2024,
       "arxiv_id": "2412.11948",
       "relevance": "Fine-tuned LLM for structured review generation, directly relevant to AI-assisted peer review."
     },
     {
       "title": "DeepReview: Improving LLM-based paper review with human-like deep thinking process",
       "authors": ["Minjun Zhu"],
       "year": 2025,
       "arxiv_id": "2503.08569",
       "relevance": "Multi-stage reasoning framework for LLM-based review generation addressing hallucinations."
     },
     {
       "title": "MARG: Multi-Agent Review Generation for Scientific Papers",
       "authors": ["Mike D'Arcy"],
       "year": 2024,
       "arxiv_id": "2401.04259",
       "relevance": "Multi-agent approach to scientific paper review generation."
     },
     {
       "title": "Dynamic LLM-Agent Network: An LLM-agent collaboration framework with agent team optimization",
       "authors": ["Zijun Liu"],
       "year": 2023,
       "arxiv_id": "2310.02170",
       "relevance": "Framework for LLM agent collaboration and team optimization relevant to multi-agent system design."
     },
     {
       "title": "Can LLMs Identify Critical Limitations within Scientific Research? A Systematic Evaluation on AI Research Papers",
       "authors": ["Zhijian Xu"],
       "year": 2025,
       "arxiv_id": "2507.02694",
       "relevance": "Systematic evaluation of LLMs for identifying research limitations in AI papers."
     }
   ]
 }