ai-research-survey

scan-v4.json (36420B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "DeepReview: Improving LLM-based Paper Review with Human-like Deep Thinking Process",
     "authors": [
       "Minjun Zhu",
       "Yixuan Weng",
       "Linyi Yang",
       "Yue Zhang"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2503.08569",
     "doi": "10.48550/arXiv.2503.08569"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 88.21% and 80.20% win rates against GPT-o1 and DeepSeek-R1 match Table 4 (ICLR 2024, Overall Judgment). The claim that DeepReviewer-14B 'outperforms CycleReviewer-70B with fewer tokens' is supported by Tables 2-3 and Section 5.5's token comparison.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claims about reasoning stages improving performance are supported by controlled ablation through Fast/Standard/Best modes (Section 5.5), which progressively add reasoning stages. The attribution of adversarial robustness to 'multi-stage reasoning framework' (Section 5.4) is more speculative but hedged with 'We attribute.'",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims to 'set a new benchmark for LLM-based paper review' broadly, but all experiments use only ICLR CS/ML papers. The title says 'Improving LLM-based Paper Review' without domain qualification. The limitations section mentions generalizability concerns but the main claims are not bounded to the tested setting.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. Potential confounds are unaddressed: the Phi-4 base model's contribution vs. the training framework, whether the LLM judge (Gemini) has systematic biases, whether training data overlap with test data inflates scores, or whether CycleReviewer baselines were run under optimal conditions.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses MSE/MAE against averaged ICLR review scores as a proxy for review quality and LLM-as-judge win rates as a proxy for review usefulness, without discussing the gap between these proxies and actual review quality. Whether matching aggregated human scores means a review is actually insightful or useful is not examined.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section discusses three specific issues: synthetic data may not capture genuine human review nuances, computational intensity of 'Best' mode, and incomplete adversarial robustness.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) 'synthetic data may not fully capture the complexities and nuances of genuine human paper review', (2) the 'Best' mode with 'complete reasoning chain and external knowledge retrieval, can be computationally intensive', (3) 'complete immunity is not yet achieved' for adversarial attacks.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound results to ICLR/CS/ML papers despite testing only on that domain. The limitations discuss general issues but do not specify excluded settings, untested populations, or claims the authors are not making.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The footnote on page 1 states 'Supported by Research Center for Industries of the Future, Westlake University.'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Zhejiang University, Westlake University, and University College London. The authors are not affiliated with any of the companies whose models they evaluate (OpenAI, Anthropic, Google, DeepSeek).",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder (Westlake University's Research Center for Industries of the Future) is an academic institution with no apparent commercial stake in the outcome of automated paper review systems.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. The absence of a disclosure statement means this criterion is not met.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "'Review quality,' 'deep thinking,' and 'reliable evaluation' are used extensively without precise operational definitions; evaluation tasks (Score, Ranking, Selection) are operationally defined but what constitutes a 'good review' is never formally specified.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly stated: DeepReview-13K dataset, DeepReviewer-14B model, and DeepReview-Bench benchmark.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Related work section engages with prior LLM review systems (AgentReview, CycleReviewer, ReviewMT) and explains how DeepReview differs by adding structured multi-stage reasoning and evidence-based argumentation.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The Resources section lists 'Code Repository: zhu-minjun/Researcher' and states 'The code, model, dataset and demo have be released in http://ai-researcher.net.' URLs are provided for models (DeepReviewer-7B, DeepReviewer-14B), dataset (DeepReview-13K), and a demo page.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The Resources section lists 'Dataset: DeepReview-13K' as released, and the paper states it will be publicly available. The underlying data is also sourced from the publicly accessible OpenReview platform.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions '8x H100 80G GPUs with DeepSpeed + ZeRO3' and training hyperparameters (Section 4.3), but provides no requirements.txt, Dockerfile, or detailed library version specifications needed to recreate the environment.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided in the paper. The methodology sections describe the pipeline conceptually but do not include specific commands, scripts, or a README-style guide to reproduce experiments.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Tables 2, 3, and 4 report only point estimates (e.g., MSE, MAE, accuracy, win rates) with no confidence intervals, error bars, or ± notation anywhere in the paper.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "The paper makes many comparative claims (e.g., '44.80% reduction in Rating MSE', '6.04% improvement in Rating Spearman') but no statistical significance tests (p-values, t-tests, bootstrap tests) are reported for any comparison.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper consistently reports improvements with baseline context, e.g., 'reduces Rating MSE by an average of 65.83%' (Section 5.2), 'improvements of 33.58% and 22.09%' in MSE and MAE vs CycleReviewer-70B (Section 5.2), and absolute numbers for both systems are visible in tables.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The test set is 10% of the dataset (1,286 samples) split by random sampling. No power analysis or explicit justification for why this sample size is adequate for the claims made.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, standard deviation, or spread measures are reported anywhere. All results in Tables 2-4 and Figure 3 appear to be single-run point estimates.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Extensive baselines are compared: prompt-based methods (AI Scientist, AgentReview) with multiple backbone LLMs (GPT-o1, Claude-3.5-sonnet, Gemini-2.0-Flash-Thinking, DeepSeek-V3, DeepSeek-R1) and fine-tuned baselines (CycleReviewer-8B, CycleReviewer-70B).",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include GPT-o1-2024-12-17, Claude-3.5-sonnet-20241022, Gemini-2.0-Flash-Thinking-01-21, DeepSeek-R1, and CycleReviewer (ICLR 2025 submission). All are state-of-the-art models from 2024-2025.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.5 presents test-time scaling analysis with Reasoning Path Scaling (Fast/Standard/Best modes, progressively adding reasoning stages z1/z2/z3) and Reviewer Scaling (R=1 to R=6), which function as ablations showing which components contribute to performance.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple metrics are used: Rating MSE, Rating MAE, Decision Accuracy, Decision F1, Rating Spearman correlation, and Pairwise Rating Accuracy (Table 2). Qualitative evaluation adds Constructive Value, Analytical Depth, Plausibility, Technical Accuracy, and Overall Judgment (Table 4).",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "The paper uses LLM-as-a-judge evaluation with Gemini-2.0-Flash-Thinking (Section 3.2, Table 4) rather than human evaluation. The appendix includes a qualitative case study comparing DeepReviewer's meta-review to real human reviews for one paper, but no systematic human evaluation of DeepReviewer's output quality is performed.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.2 states 'we randomly sampled 10% (1.2K) of the dataset to create DeepReview-Bench.' Table 1 shows separate ICLR 2024 Test (652) and ICLR 2025 Test (634) sets distinct from training data.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table 3 breaks down results by Soundness, Presentation, and Contribution dimensions. Table 4 breaks down by Constructive Value, Analytical Depth, Plausibility, Technical Accuracy, and Overall Judgment. Results are also separated by ICLR 2024 vs ICLR 2025.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No qualitative error analysis or specific failure cases are shown. The paper mentions adversarial vulnerability (0.31-point rating increase under attack in Section 5.4) and Reviewer Scaling variability, but does not analyze specific examples where DeepReviewer produces poor or incorrect reviews.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": false,
           "justification": "Every main comparison shows DeepReviewer winning. While Section 5.5 acknowledges performance variability in Reviewer Scaling when R≠4, this is presented as an expected artifact of training distribution, not a negative result. No failed approaches or configurations that underperformed are reported.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.1 specifies 'GPT-o1-2024-12-17, Claude-3.5-sonnet-20241022, Gemini-2.0-Flash-Thinking-01-21, DeepSeek-V3, and DeepSeek-R1.' Section 4.2 specifies 'Qwen-2.5-72B-Instruct', 'Qwen-2.5-3B-Instruct', and 'Phi-4 14B' for the base model.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendix Figures 4, 5, 6, and 7 provide full system prompt text for the LLM-as-judge evaluation, review improvement, paper analysis, and reliability verification stages respectively. These are complete prompt texts, not just descriptions.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.1 reports 'temperature of 0.4 with maximum input and output lengths set to 100K and 16,384 tokens.' Section 4.3 reports training: 'batch size of 16 and a learning rate of 5e-6', '23,500 steps', '256K context window using LongRoPE, with a 40K context window during training.'",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The multi-stage pipeline is described in detail in Section 4.2: Stage 1 uses Semantic Scholar API and OpenScholar for literature retrieval with ReRank for reordering; Stage 2 uses review reconstruction; Stage 3 uses Gemini for evidence analysis. The inference strategy (Section 4.3) describes how the three modes (Fast/Standard/Best) execute different subsets of the pipeline.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes collecting 18,976 submissions from OpenReview and converting with MinerU, with 'empty PDFs filtered during conversion' (footnote 2). The final dataset has 13,378 training + 1,286 test samples, leaving ~4,312 removed. The quality control mechanism is described conceptually but exact counts removed at each stage (PDF filtering, quality control failures) are not reported.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "The source data (ICLR reviews) is publicly available on OpenReview. The paper states DeepReview-13K will be released via ai-researcher.net (Resources section). The underlying ICLR review data is independently verifiable.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 describes: 'collected raw data from the OpenReview platform arXiv repository, gathering 18,976 paper submissions spanning two ICLR conference cycles (2024-2025).' Reviews include 'textual assessments (Strengths, Weaknesses, and Questions), interactive discussions from the rebuttal phase, and standardized scores.'",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants are involved in the experiments. Data comes from publicly available ICLR submissions on OpenReview. The LLM-as-judge evaluation uses an automated model, not human evaluators.",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "The pipeline stages are described (collect → convert → filter → construct reasoning chains → quality control) but exact counts at each stage are missing. Starting with 18,976 samples and ending with 14,664 (13,378 train + 1,286 test), approximately 4,312 were removed, but the paper does not break down how many were lost at PDF filtering vs. quality control.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not state the training data cutoff for Phi-4 (the base model for DeepReviewer-14B) or for any of the baseline models (GPT-o1, DeepSeek-R1, etc.). Since ICLR reviews are publicly available, these models may have seen the test data during pre-training.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not discuss whether Phi-4's pre-training data includes ICLR 2024-2025 reviews from OpenReview. The test set (DeepReview-Bench) is split from the same ICLR data, and potential overlap with base model pre-training is not analyzed.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "ICLR 2024 reviews were publicly available on OpenReview before the training cutoffs of Phi-4 and the baseline models. ICLR 2025 reviews may also have been available. The paper does not discuss this contamination risk.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved. All evaluations are automated (metrics computed on held-out data, LLM-as-judge).",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved in the experiments. The paper has an ethical considerations section but does not involve human subjects research.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved in the experiments.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved in the experiments.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved in the experiments.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved in the experiments.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants involved in the experiments.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "Section 5.5 reports approximate output token counts per mode (Fast ~3,000, Standard ~8,000, Best ~14,500) but no actual inference cost in dollars, wall-clock latency, or tokens-per-second throughput is reported. The Best mode's external API calls (Semantic Scholar, OpenScholar) add unquantified latency.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Section 4.3 mentions '8x H100 80G GPUs' and '23,500 steps with a batch size of 16' but does not report total training time in GPU-hours, wall-clock time, or total API costs for baseline evaluations.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single runs.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not state how many runs produced the results in Tables 2-4 or Figure 3. It is unclear whether results are from single runs or averaged.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "Training hyperparameters (lr=5e-6, batch=16, 23,500 steps) are reported but there is no mention of how many configurations were tried, what search method was used, or whether any hyperparameter tuning was performed.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of how the final training configuration was selected. The paper presents one configuration without explaining whether alternatives were tried or how this particular setup was chosen.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": true,
           "answer": false,
           "justification": "The paper makes dozens of comparisons across multiple metrics, models, and datasets (Tables 2-4) but performs no statistical tests at all, let alone corrections for multiple comparisons.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors implemented DeepReviewer and designed the evaluation framework (DeepReview-Bench) but do not acknowledge self-comparison bias. They also designed the data construction pipeline and the evaluation metrics, creating potential for systematic bias in their favor.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.5 and Figure 3 explicitly show performance as a function of inference token count (compute). The paper notes that 'DeepReviewer's Fast mode, with only half the output tokens (3000), outperformed the CycleReviewer model (6000 output tokens) across various metrics.'",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "The paper uses ICLR review score prediction as a benchmark for review quality without discussing whether predicting aggregated reviewer scores actually measures the ability to produce useful reviews. Whether lower MSE against averaged human scores means better review quality (vs. just matching the central tendency) is not examined.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Different baselines use fundamentally different scaffolding: AI Scientist uses agentic prompting, AgentReview uses multi-agent simulation, CycleReviewer is direct fine-tuning, and DeepReviewer has a multi-stage pipeline with external retrieval. Performance differences are attributed to the model/method without controlling for scaffolding differences.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The base model Phi-4 and baseline models (GPT-o1, DeepSeek-R1) may have been pre-trained on ICLR 2024 review data from OpenReview, which predates their release. This temporal leakage risk is not discussed.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the evaluation setup leaks information. The training and test data are from the same conferences (ICLR 2024-2025) and the model is trained to predict scores from the same review platform.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Training and test data are randomly split from the same pool of ICLR 2024-2025 submissions. Papers by the same authors, on the same topics, or from the same research groups could appear in both splits. This non-independence is not discussed.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "No leakage detection or prevention methods are employed. No canary strings, membership inference tests, decontamination pipelines, or overlap analysis between Phi-4's pre-training data and the ICLR test set.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DeepReviewer-14B reduces Rating MSE by 44.80% compared to CycleReviewer-70B on the DeepReview-Bench test set.",
       "evidence": "Table 2 shows DeepReviewer-14B achieving MSE of 1.3137/1.3410 vs CycleReviewer-70B at 2.4870/2.4294 on ICLR 2024/2025.",
       "supported": "strong"
     },
     {
       "claim": "DeepReviewer-14B achieves 88.21% and 80.20% win rates against AI Scientist (GPT-o1) and AI Scientist (DeepSeek-R1) respectively in LLM-as-judge evaluation.",
       "evidence": "Table 4 confirms these exact win rates for ICLR 2024, with Gemini-2.0-Flash-Thinking as the judge.",
       "supported": "moderate"
     },
     {
       "claim": "DeepReviewer demonstrates strong resilience to adversarial attacks despite no explicit adversarial training.",
       "evidence": "Figure 2 shows DeepReviewer's rating increases only 0.31 points under attack vs 4.26 for Gemini and 1.41 for DeepSeek-V3.",
       "supported": "moderate"
     },
     {
       "claim": "Test-time scaling (more reasoning tokens) consistently improves review quality across metrics.",
       "evidence": "Figure 3 shows positive regression trends for both Reasoning Path Scaling and Reviewer Scaling across metrics, though with variability.",
       "supported": "moderate"
     },
     {
       "claim": "DeepReviewer-14B with 14B parameters outperforms CycleReviewer-70B with 70B parameters, demonstrating parameter efficiency.",
       "evidence": "Table 2 shows DeepReviewer-14B dominating CycleReviewer-70B on all metrics; this holds across ICLR 2024 and 2025.",
       "supported": "strong"
     },
     {
       "claim": "The DeepReview-13K synthetic dataset captures the intermediate reasoning processes of academic paper reviews.",
       "evidence": "Dataset construction pipeline described in Section 4.2 using Qwen-2.5-72B-Instruct and Gemini-2.0-Flash-Thinking, but no human validation of whether synthetic reasoning matches expert reasoning is provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "DeepReviewer-14B, trained on a synthetic dataset of 13,378 structured ICLR paper reviews, substantially outperforms larger models including CycleReviewer-70B and GPT-o1 on automated rating prediction (44.80% MSE reduction) and LLM-judged review quality (88.21% win rate vs GPT-o1). The multi-stage reasoning framework shows test-time scaling behavior where increasing reasoning depth (Fast→Standard→Best modes) and simulated reviewer count improve performance. However, the evaluation relies entirely on LLM-as-judge without human validation, the model trains and tests on the same ICLR conference distribution raising distribution-match concerns, and the base model's potential pretraining overlap with test papers is unaddressed.",
   "red_flags": [
     {
       "flag": "Train/test distribution overlap",
       "detail": "Both training data (DeepReview-13K) and test data (DeepReview-Bench) come from ICLR 2024/2025 papers; the base model Phi-4 may have seen test papers in pretraining. No contamination analysis is performed."
     },
     {
       "flag": "LLM-as-judge circularity",
       "detail": "Gemini-2.0-Flash-Thinking is used as the sole judge for qualitative evaluation while also being one of the systems compared against. No human validation of review quality is conducted."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative results are point estimates with no confidence intervals, error bars, or significance tests, making it impossible to assess whether differences are meaningful."
     },
     {
       "flag": "Synthetic training data validity",
       "detail": "DeepReview-13K is generated by Qwen-2.5-72B-Instruct and Gemini-2.0-Flash-Thinking, not actual expert reviewers. The model learns to mimic LLM-generated review patterns, not genuine expert reasoning."
     },
     {
       "flag": "Missing component ablation",
       "detail": "No ablation isolates the individual contribution of z1 (novelty verification), z2 (multi-dimension review), and z3 (reliability verification); the Fast/Standard/Best comparison confounds multiple changes simultaneously."
     },
     {
       "flag": "Reward model overoptimization unaddressed",
       "detail": "The model is trained and evaluated on the same rating signal (ICLR reviewer average scores); reward hacking and overoptimization of the CycleReviewer metric are not investigated despite being a known issue in this literature."
     }
   ],
   "cited_papers": [
     {
       "title": "CycleResearcher: Improving Automated Research via Automated Review",
       "relevance": "Direct predecessor work training review models on ICLR data; DeepReviewer directly compares against CycleReviewer-70B as primary baseline."
     },
     {
       "title": "AgentReview: Exploring Peer Review Dynamics with LLM Agents",
       "relevance": "Agent-based peer review simulation; used as baseline in DeepReview experiments."
     },
     {
       "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery",
       "relevance": "Automated scientific research system used as a baseline for paper review generation."
     },
     {
       "title": "OpenScholar: Synthesizing Scientific Literature with Retrieval-Augmented LMs",
       "relevance": "Literature retrieval system used in DeepReview's novelty verification stage."
     },
     {
       "title": "Are We There Yet? Revealing the Risks of Utilizing Large Language Models in Scholarly Peer Review",
       "relevance": "Examines adversarial vulnerabilities in LLM-based review systems; motivates DeepReview's robustness evaluation."
     },
     {
       "title": "Monitoring AI-Modified Content at Scale: A Case Study on the Impact of ChatGPT on AI Conference Peer Reviews",
       "relevance": "Empirical study of LLM influence on peer review; provides context for automated review research."
     },
     {
       "title": "PeerRead: A Dataset of Peer Reviews",
       "relevance": "Early benchmark dataset for automated paper review tasks; foundational dataset in the area."
     },
     {
       "title": "Peer Review as a Multi-Turn and Long-Context Dialogue with Role-Based Interactions (ReviewMT)",
       "relevance": "Alternative approach to LLM-based review training; directly compared with DeepReview approach."
     },
     {
       "title": "Large Language Models for Automated Scholarly Paper Review: A Survey",
       "relevance": "Survey of LLM-based paper review methods providing landscape context for DeepReview's contributions."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released model and demo at ai-researcher.net/deepreviewer that researchers could use for self-assessment, though requires significant compute for the 14B model."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "A 14B model outperforming 70B models and GPT-o1 is mildly surprising, but the trend of smaller fine-tuned models beating larger general models is well-established."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises concerns about LLMs automating peer review and potential manipulation, but the paper explicitly positions itself as augmenting rather than replacing human reviewers."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Touches on the controversial topic of LLMs in peer review (ICLR 2025 introduced LLM assistance), but the paper avoids taking an adversarial position."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Live demo available at ai-researcher.net/deepreviewer with released models (DeepReviewer-7B, 14B) and code repository."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Westlake University and UCL are recognized institutions but not in the top tier of AI lab brand recognition; published at ACL."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45967079",
         "title": "Show HN: Browser-based interactive 3D Three-Body problem simulator",
         "points": 249,
         "comments": 113,
         "url": "https://news.ycombinator.com/item?id=45967079"
       },
       {
         "hn_id": "36349110",
         "title": "The Distributed Tensor Algebra Compiler (2022)",
         "points": 40,
         "comments": 6,
         "url": "https://news.ycombinator.com/item?id=36349110"
       },
       {
         "hn_id": "45202421",
         "title": "UGMM-NN: Univariate Gaussian Mixture Model Neural Network",
         "points": 31,
         "comments": 12,
         "url": "https://news.ycombinator.com/item?id=45202421"
       },
       {
         "hn_id": "44250248",
         "title": "Thermal Detection of People with Mobility Restrictions for Barrier Reduction",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44250248"
       },
       {
         "hn_id": "41139898",
         "title": "End-to-End Amp Modeling: From Data to Controllable Guitar Amplifier Models",
         "points": 2,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=41139898"
       },
       {
         "hn_id": "43772236",
         "title": "Hands-On: Segmenting Individual Signs from Continuous Sequences",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43772236"
       }
     ],
     "top_points": 249,
     "total_points": 327,
     "total_comments": 133
   }
 }