ai-research-survey

scan.json (33772B)

---

 {
   "paper": {
     "title": "DeepReview: Improving LLM-based Paper Review with Human-like Deep Thinking Process",
     "authors": [
       "Minjun Zhu",
       "Yixuan Weng",
       "Linyi Yang",
       "Yue Zhang"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2503.08569",
     "doi": "10.48550/arXiv.2503.08569"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DeepReview introduces a multi-stage review framework (novelty verification, multi-dimension evaluation, reliability verification) and trains DeepReviewer-14B on DeepReview-13K, a synthetic dataset with structured reasoning chains from ICLR 2024-2025 reviews. The 14B model achieves 44.80% lower Rating MSE than CycleReviewer-70B and 80-88% win rates against GPT-o1 and DeepSeek-R1 in LLM-as-judge evaluation. Test-time scaling through reasoning path depth (Fast/Standard/Best) and simulated reviewer count shows positive performance trends, and the model exhibits some robustness to adversarial prompt injection attacks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The Resources section lists 'Code Repository: zhu-minjun/Researcher' and states 'The code, model, dataset and demo have be released in http://ai-researcher.net.' URLs are provided for models (DeepReviewer-7B, DeepReviewer-14B), dataset (DeepReview-13K), and a demo page."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The Resources section lists 'Dataset: DeepReview-13K' as released, and the paper states it will be publicly available. The underlying data is also sourced from the publicly accessible OpenReview platform."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8x H100 80G GPUs with DeepSpeed + ZeRO3' and training hyperparameters (Section 4.3), but provides no requirements.txt, Dockerfile, or detailed library version specifications needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The methodology sections describe the pipeline conceptually but do not include specific commands, scripts, or a README-style guide to reproduce experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 3, and 4 report only point estimates (e.g., MSE, MAE, accuracy, win rates) with no confidence intervals, error bars, or ± notation anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims (e.g., '44.80% reduction in Rating MSE', '6.04% improvement in Rating Spearman') but no statistical significance tests (p-values, t-tests, bootstrap tests) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports improvements with baseline context, e.g., 'reduces Rating MSE by an average of 65.83%' (Section 5.2), 'improvements of 33.58% and 22.09%' in MSE and MAE vs CycleReviewer-70B (Section 5.2), and absolute numbers for both systems are visible in tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set is 10% of the dataset (1,286 samples) split by random sampling. No power analysis or explicit justification for why this sample size is adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported anywhere. All results in Tables 2-4 and Figure 3 appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are compared: prompt-based methods (AI Scientist, AgentReview) with multiple backbone LLMs (GPT-o1, Claude-3.5-sonnet, Gemini-2.0-Flash-Thinking, DeepSeek-V3, DeepSeek-R1) and fine-tuned baselines (CycleReviewer-8B, CycleReviewer-70B)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-o1-2024-12-17, Claude-3.5-sonnet-20241022, Gemini-2.0-Flash-Thinking-01-21, DeepSeek-R1, and CycleReviewer (ICLR 2025 submission). All are state-of-the-art models from 2024-2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 presents test-time scaling analysis with Reasoning Path Scaling (Fast/Standard/Best modes, progressively adding reasoning stages z1/z2/z3) and Reviewer Scaling (R=1 to R=6), which function as ablations showing which components contribute to performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Rating MSE, Rating MAE, Decision Accuracy, Decision F1, Rating Spearman correlation, and Pairwise Rating Accuracy (Table 2). Qualitative evaluation adds Constructive Value, Analytical Depth, Plausibility, Technical Accuracy, and Overall Judgment (Table 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LLM-as-a-judge evaluation with Gemini-2.0-Flash-Thinking (Section 3.2, Table 4) rather than human evaluation. The appendix includes a qualitative case study comparing DeepReviewer's meta-review to real human reviews for one paper, but no systematic human evaluation of DeepReviewer's output quality is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states 'we randomly sampled 10% (1.2K) of the dataset to create DeepReview-Bench.' Table 1 shows separate ICLR 2024 Test (652) and ICLR 2025 Test (634) sets distinct from training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 breaks down results by Soundness, Presentation, and Contribution dimensions. Table 4 breaks down by Constructive Value, Analytical Depth, Plausibility, Technical Accuracy, and Overall Judgment. Results are also separated by ICLR 2024 vs ICLR 2025."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative error analysis or specific failure cases are shown. The paper mentions adversarial vulnerability (0.31-point rating increase under attack in Section 5.4) and Reviewer Scaling variability, but does not analyze specific examples where DeepReviewer produces poor or incorrect reviews."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every main comparison shows DeepReviewer winning. While Section 5.5 acknowledges performance variability in Reviewer Scaling when R≠4, this is presented as an expected artifact of training distribution, not a negative result. No failed approaches or configurations that underperformed are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 88.21% and 80.20% win rates against GPT-o1 and DeepSeek-R1 match Table 4 (ICLR 2024, Overall Judgment). The claim that DeepReviewer-14B 'outperforms CycleReviewer-70B with fewer tokens' is supported by Tables 2-3 and Section 5.5's token comparison."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claims about reasoning stages improving performance are supported by controlled ablation through Fast/Standard/Best modes (Section 5.5), which progressively add reasoning stages. The attribution of adversarial robustness to 'multi-stage reasoning framework' (Section 5.4) is more speculative but hedged with 'We attribute.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims to 'set a new benchmark for LLM-based paper review' broadly, but all experiments use only ICLR CS/ML papers. The title says 'Improving LLM-based Paper Review' without domain qualification. The limitations section mentions generalizability concerns but the main claims are not bounded to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. Potential confounds are unaddressed: the Phi-4 base model's contribution vs. the training framework, whether the LLM judge (Gemini) has systematic biases, whether training data overlap with test data inflates scores, or whether CycleReviewer baselines were run under optimal conditions."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses MSE/MAE against averaged ICLR review scores as a proxy for review quality and LLM-as-judge win rates as a proxy for review usefulness, without discussing the gap between these proxies and actual review quality. Whether matching aggregated human scores means a review is actually insightful or useful is not examined."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 specifies 'GPT-o1-2024-12-17, Claude-3.5-sonnet-20241022, Gemini-2.0-Flash-Thinking-01-21, DeepSeek-V3, and DeepSeek-R1.' Section 4.2 specifies 'Qwen-2.5-72B-Instruct', 'Qwen-2.5-3B-Instruct', and 'Phi-4 14B' for the base model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix Figures 4, 5, 6, and 7 provide full system prompt text for the LLM-as-judge evaluation, review improvement, paper analysis, and reliability verification stages respectively. These are complete prompt texts, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports 'temperature of 0.4 with maximum input and output lengths set to 100K and 16,384 tokens.' Section 4.3 reports training: 'batch size of 16 and a learning rate of 5e-6', '23,500 steps', '256K context window using LongRoPE, with a 40K context window during training.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-stage pipeline is described in detail in Section 4.2: Stage 1 uses Semantic Scholar API and OpenScholar for literature retrieval with ReRank for reordering; Stage 2 uses review reconstruction; Stage 3 uses Gemini for evidence analysis. The inference strategy (Section 4.3) describes how the three modes (Fast/Standard/Best) execute different subsets of the pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes collecting 18,976 submissions from OpenReview and converting with MinerU, with 'empty PDFs filtered during conversion' (footnote 2). The final dataset has 13,378 training + 1,286 test samples, leaving ~4,312 removed. The quality control mechanism is described conceptually but exact counts removed at each stage (PDF filtering, quality control failures) are not reported."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section discusses three specific issues: synthetic data may not capture genuine human review nuances, computational intensity of 'Best' mode, and incomplete adversarial robustness."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) 'synthetic data may not fully capture the complexities and nuances of genuine human paper review', (2) the 'Best' mode with 'complete reasoning chain and external knowledge retrieval, can be computationally intensive', (3) 'complete immunity is not yet achieved' for adversarial attacks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound results to ICLR/CS/ML papers despite testing only on that domain. The limitations discuss general issues but do not specify excluded settings, untested populations, or claims the authors are not making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The source data (ICLR reviews) is publicly available on OpenReview. The paper states DeepReview-13K will be released via ai-researcher.net (Resources section). The underlying ICLR review data is independently verifiable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes: 'collected raw data from the OpenReview platform arXiv repository, gathering 18,976 paper submissions spanning two ICLR conference cycles (2024-2025).' Reviews include 'textual assessments (Strengths, Weaknesses, and Questions), interactive discussions from the rebuttal phase, and standardized scores.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the experiments. Data comes from publicly available ICLR submissions on OpenReview. The LLM-as-judge evaluation uses an automated model, not human evaluators."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline stages are described (collect → convert → filter → construct reasoning chains → quality control) but exact counts at each stage are missing. Starting with 18,976 samples and ending with 14,664 (13,378 train + 1,286 test), approximately 4,312 were removed, but the paper does not break down how many were lost at PDF filtering vs. quality control."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The footnote on page 1 states 'Supported by Research Center for Industries of the Future, Westlake University.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Zhejiang University, Westlake University, and University College London. The authors are not affiliated with any of the companies whose models they evaluate (OpenAI, Anthropic, Google, DeepSeek)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (Westlake University's Research Center for Industries of the Future) is an academic institution with no apparent commercial stake in the outcome of automated paper review systems."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. The absence of a disclosure statement means this criterion is not met."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff for Phi-4 (the base model for DeepReviewer-14B) or for any of the baseline models (GPT-o1, DeepSeek-R1, etc.). Since ICLR reviews are publicly available, these models may have seen the test data during pre-training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Phi-4's pre-training data includes ICLR 2024-2025 reviews from OpenReview. The test set (DeepReview-Bench) is split from the same ICLR data, and potential overlap with base model pre-training is not analyzed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ICLR 2024 reviews were publicly available on OpenReview before the training cutoffs of Phi-4 and the baseline models. ICLR 2025 reviews may also have been available. The paper does not discuss this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved. All evaluations are automated (metrics computed on held-out data, LLM-as-judge)."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the experiments. The paper has an ethical considerations section but does not involve human subjects research."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the experiments."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the experiments."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the experiments."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the experiments."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in the experiments."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.5 reports approximate output token counts per mode (Fast ~3,000, Standard ~8,000, Best ~14,500) but no actual inference cost in dollars, wall-clock latency, or tokens-per-second throughput is reported. The Best mode's external API calls (Semantic Scholar, OpenScholar) add unquantified latency."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 mentions '8x H100 80G GPUs' and '23,500 steps with a batch size of 16' but does not report total training time in GPU-hours, wall-clock time, or total API costs for baseline evaluations."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many runs produced the results in Tables 2-4 or Figure 3. It is unclear whether results are from single runs or averaged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Training hyperparameters (lr=5e-6, batch=16, 23,500 steps) are reported but there is no mention of how many configurations were tried, what search method was used, or whether any hyperparameter tuning was performed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the final training configuration was selected. The paper presents one configuration without explaining whether alternatives were tried or how this particular setup was chosen."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes dozens of comparisons across multiple metrics, models, and datasets (Tables 2-4) but performs no statistical tests at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implemented DeepReviewer and designed the evaluation framework (DeepReview-Bench) but do not acknowledge self-comparison bias. They also designed the data construction pipeline and the evaluation metrics, creating potential for systematic bias in their favor."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 and Figure 3 explicitly show performance as a function of inference token count (compute). The paper notes that 'DeepReviewer's Fast mode, with only half the output tokens (3000), outperformed the CycleReviewer model (6000 output tokens) across various metrics.'"
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses ICLR review score prediction as a benchmark for review quality without discussing whether predicting aggregated reviewer scores actually measures the ability to produce useful reviews. Whether lower MSE against averaged human scores means better review quality (vs. just matching the central tendency) is not examined."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Different baselines use fundamentally different scaffolding: AI Scientist uses agentic prompting, AgentReview uses multi-agent simulation, CycleReviewer is direct fine-tuning, and DeepReviewer has a multi-stage pipeline with external retrieval. Performance differences are attributed to the model/method without controlling for scaffolding differences."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The base model Phi-4 and baseline models (GPT-o1, DeepSeek-R1) may have been pre-trained on ICLR 2024 review data from OpenReview, which predates their release. This temporal leakage risk is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The training and test data are from the same conferences (ICLR 2024-2025) and the model is trained to predict scores from the same review platform."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training and test data are randomly split from the same pool of ICLR 2024-2025 submissions. Papers by the same authors, on the same topics, or from the same research groups could appear in both splits. This non-independence is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are employed. No canary strings, membership inference tests, decontamination pipelines, or overlap analysis between Phi-4's pre-training data and the ICLR test set."
       }
     }
   },
   "claims": [
     {
       "claim": "DeepReviewer-14B reduces Rating MSE by 44.80% compared to CycleReviewer-70B",
       "evidence": "Table 2 shows DeepReviewer-14B achieves Rating MSE of 1.3137 (ICLR 2024) vs CycleReviewer-70B's 2.4870, and 1.3410 vs 2.4294 on ICLR 2025.",
       "supported": "moderate"
     },
     {
       "claim": "DeepReviewer-14B achieves 88.21% win rate against GPT-o1 and 80.20% against DeepSeek-R1 in overall judgment",
       "evidence": "Table 4 (ICLR 2024) shows Overall Judgment win rates of 88.21% vs AI Scientist GPT-o1 and 80.20% vs AI Scientist DeepSeek-R1, using Gemini-2.0-Flash-Thinking as judge.",
       "supported": "moderate"
     },
     {
       "claim": "DeepReviewer shows strong resilience to adversarial attacks with only 0.31 rating increase under attack",
       "evidence": "Figure 2 (Section 5.4) shows overall rating increase of 0.31 points (5.38→5.69) vs Gemini's 4.26 points (4.23→8.49). Attribution to multi-stage reasoning is speculative.",
       "supported": "moderate"
     },
     {
       "claim": "Test-time scaling through reasoning path and reviewer count improves performance",
       "evidence": "Figure 3 (Section 5.5) shows positive trends in Decision Accuracy, Rating MSE, Spearman correlation as tokens increase from Fast→Standard→Best modes, and from R=1 to R=6 reviewers.",
       "supported": "moderate"
     },
     {
       "claim": "DeepReviewer's Fast mode outperforms CycleReviewer with fewer output tokens",
       "evidence": "Section 5.5 states Fast mode (3,000 tokens) outperforms CycleReviewer (6,000 tokens) on Decision Accuracy, Rating MSE, and fine-grained Spearman correlations, supported by Table 3 comparisons.",
       "supported": "moderate"
     },
     {
       "claim": "DeepReviewer improves Rating Spearman by 6.04% over CycleReviewer-70B",
       "evidence": "Table 2 shows Rating Spearman of 0.3559 vs 0.3356 (ICLR 2024) and 0.4047 vs 0.2674 (ICLR 2025). The 6.04% improvement is for ICLR 2024; ICLR 2025 shows a much larger gap.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "LLM-as-judge circularity",
       "detail": "Gemini-2.0-Flash-Thinking serves as the evaluation judge (Table 4) but was also used in the data construction pipeline for paper analysis (Stage 1, Figure 6) and reliability verification (Stage 3, Figure 7). The judge may systematically favor outputs whose reasoning style it helped shape, creating circular validation."
     },
     {
       "flag": "No human evaluation of review quality",
       "detail": "For a paper about automating paper review, the absence of systematic human evaluation is notable. All quality assessments rely on automated metrics (MSE against aggregated scores) or an LLM judge. Whether DeepReviewer's reviews are actually useful to authors or area chairs is not tested."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All claims of superiority over baselines are based on comparing point estimates across Tables 2-4 without any significance tests, error bars, or variance measures. With many comparisons across models, metrics, and datasets, some apparent improvements could be noise."
     },
     {
       "flag": "Self-designed benchmark",
       "detail": "The authors created DeepReview-Bench from the same data distribution as their training set (ICLR 2024-2025 via OpenReview), designed the evaluation metrics, and trained the model — creating potential for optimization toward their own benchmark rather than genuine review capability."
     },
     {
       "flag": "Contamination risk from base model pre-training",
       "detail": "Phi-4 and all baseline models (GPT-o1, DeepSeek-R1, etc.) may have been pre-trained on ICLR reviews from OpenReview, which is the source of both training and test data. The paper does not investigate or discuss this overlap."
     },
     {
       "flag": "Overclaiming scope",
       "detail": "The abstract claims to 'set a new benchmark for LLM-based paper review' and the title says 'Improving LLM-based Paper Review' broadly, but all experiments are limited to ICLR CS/ML papers. Generalization to other venues, disciplines, or paper types is untested."
     }
   ],
   "cited_papers": [
     {
       "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery",
       "authors": ["Chris Lu", "Cong Lu", "Robert Tjarko Lange", "Jakob Foerster", "Jeff Clune", "David Ha"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Foundational work on autonomous AI-driven scientific discovery using LLM agents, directly compared as a baseline."
     },
     {
       "title": "AgentReview: Exploring Peer Review Dynamics with LLM Agents",
       "authors": ["Yuxuan Jin", "Qinlin Zhao", "Yilun Wang"],
       "year": 2024,
       "relevance": "Multi-agent LLM framework simulating peer review dynamics, used as a prompt-based baseline in DeepReview's evaluation."
     },
     {
       "title": "CycleResearcher: Improving Automated Research via Automated Review",
       "authors": ["Yixuan Weng", "Minjun Zhu", "Guanghao Bao"],
       "year": 2025,
       "relevance": "Closely related work training LLM-based research and review models with reinforcement learning; CycleReviewer-70B is a primary baseline."
     },
     {
       "title": "OpenScholar: Synthesizing Scientific Literature with Retrieval-Augmented LMs",
       "authors": ["Akari Asai"],
       "year": 2024,
       "arxiv_id": "2411.14199",
       "relevance": "RAG system for scientific literature synthesis, used as a core component in DeepReview's novelty verification stage."
     },
     {
       "title": "Are We There Yet? Revealing the Risks of Utilizing Large Language Models in Scholarly Peer Review",
       "authors": ["Ruofan Ye"],
       "year": 2024,
       "arxiv_id": "2412.01708",
       "relevance": "Examines risks and vulnerabilities of LLM-based peer review including prompt engineering attacks, directly relevant to DeepReview's adversarial evaluation."
     },
     {
       "title": "LLMs Assist NLP Researchers: Critique Paper (Meta-)Reviewing",
       "authors": ["Jiangshu Du"],
       "year": 2024,
       "doi": "10.18653/v1/2024.emnlp-main.292",
       "relevance": "LLM-assisted paper review and meta-review generation, directly in the automated review domain."
     },
     {
       "title": "Can LLMs Generate Novel Research Ideas? A Large-Scale Human Study with 100+ NLP Researchers",
       "authors": ["Chenglei Si", "Diyi Yang", "Tatsunori Hashimoto"],
       "year": 2025,
       "relevance": "Large-scale evaluation of LLM capability for scientific idea generation with human expert assessment."
     },
     {
       "title": "The AI Review Lottery: Widespread AI-Assisted Peer Reviews Boost Paper Scores and Acceptance Rates",
       "authors": ["Gabriel R. Latona"],
       "year": 2024,
       "arxiv_id": "2405.02150",
       "relevance": "Empirical study showing AI-assisted reviews systematically boost scores, raising concerns about automated review reliability."
     },
     {
       "title": "Potential and Perils of Large Language Models as Judges of Unstructured Textual Data",
       "authors": ["Rewina Bedemariam"],
       "year": 2025,
       "arxiv_id": "2501.08167",
       "relevance": "Evaluates reliability and limitations of LLM-as-judge paradigm, directly relevant to DeepReview's evaluation methodology."
     },
     {
       "title": "Large Language Models for Automated Scholarly Paper Review: A Survey",
       "authors": ["Zhenzhen Zhuang"],
       "year": 2025,
       "relevance": "Comprehensive survey of the LLM-based paper review landscape that DeepReview contributes to."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan"],
       "year": 2024,
       "relevance": "Self-refinement technique for LLMs that influences DeepReview's reliability verification stage design."
     },
     {
       "title": "Peer Review as a Multi-Turn and Long-Context Dialogue with Role-Based Interactions",
       "authors": ["Chao Tan"],
       "year": 2024,
       "arxiv_id": "2406.05688",
       "relevance": "Alternative approach to LLM-based review modeling as multi-turn dialogue, used as a comparison point for DeepReview."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released model and demo at ai-researcher.net/deepreviewer that researchers could use for self-assessment, though requires significant compute for the 14B model."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "A 14B model outperforming 70B models and GPT-o1 is mildly surprising, but the trend of smaller fine-tuned models beating larger general models is well-established."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns about LLMs automating peer review and potential manipulation, but the paper explicitly positions itself as augmenting rather than replacing human reviewers."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Touches on the controversial topic of LLMs in peer review (ICLR 2025 introduced LLM assistance), but the paper avoids taking an adversarial position."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Live demo available at ai-researcher.net/deepreviewer with released models (DeepReviewer-7B, 14B) and code repository."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Westlake University and UCL are recognized institutions but not in the top tier of AI lab brand recognition; published at ACL."
     }
   }
 }