ai-research-survey

scan.json (27565B)

---

 {
   "paper": {
     "title": "AI-Driven Scholarly Peer Review via Persistent Workflow Prompting, Meta-Prompting, and Meta-Reasoning",
     "authors": ["Evgeny Markhasin"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2505.03332",
     "doi": "10.48550/arXiv.2505.03332"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The full PeerReviewPrompt.md and other prompt files are provided via an OSF repository (https://osf.io/nq68y/) and as PDF attachments. While there is no traditional code, the prompt artifacts constitute the core reproducible artifact of this work."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The test paper used for demonstrations is provided via a view-only OSF link (Supporting Information Section B), and shared AI chat logs are linked throughout the paper (e.g., references [84], [87], [88], [90], [91], [101], [102], [103])."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the primary target model is 'Gemini Advanced 2.5 Pro' and mentions 'ChatGPT Plus o1 & o3' and 'SuperGrok Grok 3 Think', but no version snapshots, API versions, or dates of access beyond 'as of Apr 2025' are provided. No environment specifications (software versions, dependencies) are given since the approach uses standard chat interfaces."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section C of the Supporting Information provides a 'Demo Usage Protocol for PeerReviewPrompt' with step-by-step instructions: submit the prompt as Message 1, then submit a specific analysis request with the manuscript attached as Message 2, plus sample prompts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents only qualitative demonstrations with no quantitative results or metrics, so confidence intervals are structurally inapplicable."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative quantitative claims are made that would require significance tests. The paper is a qualitative proof-of-concept."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative experiments or measured effects are reported. The evaluation is entirely qualitative."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses a single test case (one chemistry paper) for development and demonstration. Section 3.4 acknowledges this as a limitation ('Single Test Case') but provides no justification for why a single paper is sufficient, nor any power analysis. The limitation is merely acknowledged, not justified."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative experiments with multiple runs are reported. The paper mentions that 'specific details and phrasing of the analyses varied between models and even between different runs' (Section 3) but does not quantify this variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a model might identify potential flaws in an improvised experimental setup but still conclude the experiment was successful based solely on the manuscript's claim' as an example of naive analysis behavior, and mentions a 'Gemini - Baseline' demonstration in the appendix. However, no systematic comparison against baseline prompting techniques is presented. Section 3.4 explicitly acknowledges: 'No quantitative benchmark was constructed for systematic evaluation against ground truth or objective metrics' and the evaluation 'lacks comparison to defined baselines.'"
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are systematically compared against, so contemporaneity cannot be assessed. The paper cites many related AI peer review systems but does not compare PWP against any of them."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The PWP prompt has multiple components (persona engineering, workflow library, hierarchical structure, bias mitigation), but no ablation study tests the contribution of individual components to the overall performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative evaluation metrics are used at all. The evaluation is purely qualitative and observational, as acknowledged in Section 3.4: 'No quantitative benchmark was constructed for systematic evaluation.'"
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper's evaluation consists only of the author's own informal assessment of the LLM outputs. Section 3.4 states: 'Performance-related statements are based solely on the author's conventional (human-driven) evaluation of the generated LLM analyses, which introduces subjectivity.' This is not a structured human evaluation with defined criteria or multiple evaluators."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The single test paper was used for both developing and evaluating the prompt. Section 3.4 states: 'The PeerReviewPrompt was developed and primarily tested using a single publication.' There is no held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No per-category or per-task breakdown of performance is provided. Results are discussed as qualitative anecdotes from demonstration analyses."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 discusses several limitations and failure modes: LLM hallucinations, inconsistent context recall, platform compatibility issues (Qwen rejecting the prompt due to input limits), and the general unreliability of multimodal analysis in some models. Section 3.1 notes that 'verifying the extent of such capabilities in ChatGPT models can be challenging.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that early simpler versions of the prompt exhibited positive input bias (Section 3.2), that ChatGPT models did not yield convincing multimodal analysis (Section 3.1), and that the Qwen chat interface rejected the prompt entirely due to input limits (Section 3.4). These are things that did not work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the PWP prompt 'equips the LLM with persistent workflows triggered by subsequent queries, guiding modern reasoning LLMs through systematic, multimodal evaluations' and that 'Demonstrations show the PWP-guided LLM identifying major methodological flaws.' These claims are supported only by qualitative demonstrations on a single test case chosen specifically for its known flaws, not by systematic evaluation. The abstract presents the results more strongly than the evidence warrants."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'the negative bias conditioning implemented in the current PeerReviewPrompt successfully and reliably suppressed the observed positive input bias' (Section 3.2) and that PWP 'effectively directs LLM reasoning towards critical evaluation points' (Section 3). These causal claims are based on informal observation of a single test case without controlled experiments to isolate the effect of the prompt engineering techniques."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.3 explicitly states 'Our investigation is deliberately constrained to using frontier LLMs accessible via standard chat interfaces.' Section 3.4 acknowledges limitations including single test case, limited prompt scope, and qualitative non-benchmarked evaluation. The paper consistently frames itself as a 'proof-of-concept' throughout."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 discusses alternative explanations for why PWP-guided LLMs can identify flaws, suggesting it may arise 'not from deep reasoning in the human sense, but from a sophisticated comparison of manuscript-specific details against learned representations of scientific norms and consensus.' This is a substantive discussion of an alternative mechanism."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Gemini Advanced 2.5 Pro', 'ChatGPT Plus o1', 'ChatGPT Plus o3', and 'SuperGrok Grok 3 Think'. These are marketing names without specific version snapshots or API version identifiers. The only temporal qualifier is 'as of Apr 2025' in the feature highlights appendix."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full PeerReviewPrompt text is provided in Appendix A (approximately 10 pages of detailed prompt text), and is also available as a Markdown file via the OSF repository. Additional meta-prompts are provided as well. This is exemplary prompt transparency."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM hyperparameters are reported anywhere in the paper. The paper uses standard chat interfaces, but these interfaces still have default settings that affect output."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The approach uses no agentic scaffolding — it operates entirely within standard LLM chat interfaces with no API calls, tool use, or external systems. The prompt itself is the entire system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2.1 states: 'the input material used was the manuscript file combined with its corresponding supporting information, taken exactly as provided by the publisher without structural modification or reformatting.' This documents that no preprocessing was applied."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 is titled 'Study Limitations' and contains five detailed, numbered limitation points covering single test case, limited prompt scope, qualitative evaluation, prompt size constraints, and uncharacterized LLM reliability."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 discusses specific threats: (1) the single test case was chosen for its known flaws, which limits generalizability to sound papers; (2) the evaluation is 'based solely on the author's conventional (human-driven) evaluation, which introduces subjectivity'; (3) the prompt exceeds input limits on some platforms (specific example: Qwen's 10,000 character limit). These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.3 explicitly states scope boundaries: 'deliberately constrained to using frontier LLMs accessible via standard chat interfaces, without relying on APIs, coding, or specialized tools.' Section 3.4 identifies what was NOT tested: 'testing on a broader range of manuscripts remains future work', evaluation on methodologically sound papers, and quantitative benchmarking."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The shared AI chat logs (linked throughout as references [84], [87], [88], [90], [91], [95], [96], [101], [102], [103]) constitute the raw data of this study, and they are publicly accessible via shared links to Gemini, ChatGPT, and Grok conversations."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the iterative development process in detail across Sections 2.1-2.3, including which meta-prompting techniques were used, which models were involved, and the sequence of refinement. The demonstration protocol is specified in Supporting Information Section C."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. The study uses LLMs and a single test paper selected by the author."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from prompt development (meta-prompting) through demonstration (submitting prompt + paper to LLM) to evaluation (author assessment of outputs) is described in the methodology and results sections. The process is straightforward: submit prompt, submit paper, receive analysis."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgment of funding is disclosed. The Acknowledgments section mentions only the use of generative AI in writing. For a university-affiliated author, the absence of any funding disclosure (even 'no funding') is notable."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation with Lobachevsky State University of Nizhny Novgorod is clearly stated, along with ORCID and LinkedIn links. The paper evaluates no specific commercial product, so there is no product-affiliation conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper does not state whether it is unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate model capability on a benchmark. It tests whether a prompt engineering technique can guide LLMs to produce useful peer review output, not whether models have memorized specific content."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is conducted. The paper tests a prompt engineering methodology, not model knowledge on a standard benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark is used. The paper evaluates LLM-generated analysis quality qualitatively on a single test paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report inference cost, token consumption, or wall-clock time for running the PWP prompt. Given that the prompt exceeds 30 kB of text and is used with subscription-based services (Gemini Advanced, ChatGPT Plus), practical cost information would be relevant but is absent."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper uses commercial chat interfaces (Gemini Advanced, ChatGPT Plus, SuperGrok) but does not quantify the total resources consumed."
       }
     }
   },
   "claims": [
     {
       "claim": "PWP-guided LLMs can identify major methodological flaws in scientific manuscripts while mitigating LLM input bias.",
       "evidence": "Section 3.1 describes demonstrations where multiple frontier LLMs guided by the PeerReviewPrompt 'relatively reliably identified major methodological flaws within the test paper and converged on the conclusion that its central claim was highly dubious.' Section 3.2 discusses bias mitigation. Evidence is based on qualitative analysis of a single test paper with known flaws.",
       "supported": "weak"
     },
     {
       "claim": "The PWP prompt successfully counteracts default positive input biases in LLMs, enabling more critical and objective analysis.",
       "evidence": "Section 3.2 states 'the negative bias conditioning implemented in the current PeerReviewPrompt successfully and reliably suppressed the observed positive input bias when applied to the test paper using the target models.' However, this is demonstrated only qualitatively on one paper, with no systematic measurement of bias reduction.",
       "supported": "weak"
     },
     {
       "claim": "PWP-guided LLMs identified issues initially overlooked during manual human review, including the glass-wool-packed condenser criticism and boiling point inconsistencies.",
       "evidence": "Section 3.1 describes these findings, noting 'the LLM analyses highlighted at least two potentially significant issues not initially noted by the author during manual review.' The author acknowledges these fall outside direct expertise and preliminary external checks suggest validity.",
       "supported": "moderate"
     },
     {
       "claim": "The consistency of flaw identification across different model architectures suggests PWP effectively directs LLM reasoning toward critical evaluation points.",
       "evidence": "Section 3 states 'all tested models, when guided by the PeerReviewPrompt, relatively reliably identified major methodological flaws.' However, no quantitative consistency metrics are reported, and the test paper was selected specifically for having known flaws.",
       "supported": "weak"
     },
     {
       "claim": "PWP is a broadly applicable prompt engineering methodology that can be adapted to complex analytical tasks beyond chemistry peer review.",
       "evidence": "The paper argues this throughout (Section 1.2, 3.5, 4) but provides no evidence beyond the single chemistry demonstration. This claim is aspirational rather than empirically supported.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "The paper introduces Persistent Workflow Prompting (PWP), a prompt engineering methodology that uses a large, hierarchically structured prompt as a persistent workflow library for LLM chat interfaces. Applied to experimental chemistry peer review, the approach demonstrated that multiple frontier LLMs (Gemini 2.5 Pro, ChatGPT o1/o3, Grok 3) could identify known methodological flaws in a test paper and that persona engineering with negative bias conditioning could counteract LLMs' default tendency toward positive input bias. However, all findings are based on qualitative analysis of a single deliberately chosen flawed test case, with no quantitative evaluation, no baselines, and no systematic benchmarking.",
   "red_flags": [
     {
       "flag": "Single test case used for both development and evaluation",
       "detail": "The PeerReviewPrompt was developed and evaluated using the same single paper (Prasad et al. 2011), which was deliberately chosen for its known methodological flaws. This means the prompt was essentially optimized to find flaws in a paper known to have them, providing no evidence of generalizability to methodologically sound papers, papers with subtle flaws, or papers in different chemistry subdisciplines."
     },
     {
       "flag": "No quantitative evaluation or benchmarking",
       "detail": "Section 3.4 explicitly acknowledges: 'No quantitative benchmark was constructed for systematic evaluation against ground truth or objective metrics. Performance-related statements are based solely on the author's conventional (human-driven) evaluation of the generated LLM analyses, which introduces subjectivity and lacks comparison to defined baselines.' Despite the 105-reference bibliography, the paper's empirical contribution rests on anecdotal observations."
     },
     {
       "flag": "Circular evaluation",
       "detail": "The author selected a paper with known flaws, designed a prompt to find those types of flaws, then evaluated success by whether the LLM found those flaws. The author's own assessment is the only evaluation criterion, creating a circular validation loop. No independent evaluator was involved."
     },
     {
       "flag": "Broad generalizability claims from single proof-of-concept",
       "detail": "The paper claims PWP is 'intended for broader applicability across various complex analytical tasks' and discusses adaptation to 'physics, biology, materials science, computer science' (Section 3.5). These claims have zero empirical support beyond one chemistry test case."
     },
     {
       "flag": "No comparison to simpler prompting approaches",
       "detail": "The paper does not systematically compare the 30+ kB PWP prompt against simpler alternatives (e.g., a one-paragraph instruction to critically review the paper, chain-of-thought prompting, or existing peer review prompts). Without such comparisons, the marginal value of the elaborate prompt architecture is unknown."
     }
   ],
   "cited_papers": [
     {
       "title": "AI-Driven Review Systems: Evaluating LLMs in Scalable and Bias-Aware Academic Reviews",
       "authors": ["K. Tyser", "B. Segev", "G. Longhitano"],
       "year": 2024,
       "arxiv_id": "2408.10365",
       "relevance": "Directly relevant as a system for AI-driven academic peer review, addressing scalability and bias in LLM reviews."
     },
     {
       "title": "Are We There Yet? Revealing the Risks of Utilizing Large Language Models in Scholarly Peer Review",
       "authors": ["R. Ye", "X. Pang", "J. Chai"],
       "year": 2024,
       "arxiv_id": "2412.01708",
       "relevance": "Evaluates risks and limitations of using LLMs for peer review, directly relevant to assessing AI-assisted review quality."
     },
     {
       "title": "Can large language models provide useful feedback on research papers? A large-scale empirical analysis",
       "authors": ["W. Liang", "Y. Zhang", "H. Cao"],
       "year": 2023,
       "arxiv_id": "2310.01783",
       "relevance": "Large-scale empirical study of LLM feedback on research papers, relevant to evaluating LLM review capabilities."
     },
     {
       "title": "ReviewerGPT? An Exploratory Study on Using Large Language Models for Paper Reviewing",
       "authors": ["R. Liu", "N.B. Shah"],
       "year": 2023,
       "arxiv_id": "2306.00622",
       "relevance": "Exploratory study on LLMs for paper reviewing, directly relevant to AI-assisted peer review quality assessment."
     },
     {
       "title": "Meta-Prompting: Enhancing Language Models with Task-Agnostic Scaffolding",
       "authors": ["M. Suzgun", "A.T. Kalai"],
       "year": 2024,
       "arxiv_id": "2401.12954",
       "relevance": "Foundational work on meta-prompting techniques that this paper builds upon for prompt engineering methodology."
     },
     {
       "title": "The Prompt Report: A Systematic Survey of Prompt Engineering Techniques",
       "authors": ["S. Schulhoff", "M. Ilie", "N. Balepur"],
       "year": 2025,
       "arxiv_id": "2406.06608",
       "relevance": "Comprehensive survey of prompt engineering techniques, providing methodological context for the PWP approach."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans"],
       "year": 2023,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting technique that PWP builds upon for guiding LLM reasoning through structured steps."
     },
     {
       "title": "DeepReview: Improving LLM-based Paper Review with Human-like Deep Thinking Process",
       "authors": ["M. Zhu", "Y. Weng", "L. Yang"],
       "year": 2025,
       "arxiv_id": "2503.08569",
       "relevance": "Recent work on improving LLM-based paper review through structured reasoning, directly comparable to PWP approach."
     },
     {
       "title": "A Peek into Token Bias: Large Language Models Are Not Yet Genuine Reasoners",
       "authors": ["B. Jiang", "Y. Xie", "Z. Hao"],
       "year": 2024,
       "arxiv_id": "2406.11050",
       "relevance": "Relevant to understanding LLM reasoning biases that PWP aims to mitigate through prompt engineering."
     },
     {
       "title": "Automatically Evaluating the Paper Reviewing Capability of Large Language Models",
       "authors": ["H. Shin", "J. Tang", "Y. Lee"],
       "year": 2025,
       "arxiv_id": "2502.17086",
       "relevance": "Provides methods for evaluating LLM reviewing capability, relevant to benchmarking AI peer review systems."
     },
     {
       "title": "CycleResearcher: Improving Automated Research via Automated Review",
       "authors": ["Y. Weng", "M. Zhu", "G. Bao"],
       "year": 2025,
       "arxiv_id": "2411.00816",
       "relevance": "Develops automated review systems with training datasets, relevant to AI-driven scholarly review infrastructure."
     }
   ]
 }