ai-research-survey

scan.json (27299B)

---

 {
   "paper": {
     "title": "AInstein: Assessing the Feasibility of AI-Generated Approaches to Research Problems",
     "authors": [
       "Shambhavi Mishra",
       "Gaurav Sahu",
       "Marco Pedersoli",
       "Laurent Charlin",
       "Jose Dolz",
       "Christopher Pal"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.05432"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. There is no mention of code being available."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a curated dataset of 1,214 ICLR 2025 papers but does not provide a download link or release the curated set, problem statements, or generated solutions."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment setup, dependency list, or software versions are specified. The paper does not include a requirements.txt, Dockerfile, or equivalent."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step instructions for reproducing the experiments are provided. The methodology is described at a high level but without specific runnable commands or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2 and 6 report mean ± std for various metrics (e.g., '8.80 ± 0.77' for Fidelity, '0.868 ± 0.035' for cosine similarity). Standard deviations serve as a measure of uncertainty."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B (Table 7) reports two-sample t-tests and Mann-Whitney U tests with Bonferroni correction (α ≈ 0.0036) comparing GPT-OSS-120B and Mistral-24B performance, including p-values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 in Appendix B reports Cohen's d for each comparison (e.g., d = 0.021, d = -0.033, d = 0.070), providing standardized effect size measures alongside significance tests."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 1,214 papers with intentional oversampling of Oral and Spotlight tiers (Table 1) but provides no formal justification for why this sample size is sufficient or why this specific stratification ratio was chosen. No power analysis is presented."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in Tables 2 and 6 (e.g., '8.80 ± 0.77'). These indicate variance across the experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares three model families (GPT-OSS-120B, Qwen-235B, Mistral-24B) against each other and also includes 'Human Abstracts' as a gold-standard reference in the ELO tournament (Table 5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All three models tested (GPT-OSS-120B, Qwen-235B, Mistral-24B) are current-generation large language models. The human abstracts from ICLR 2025 serve as the primary comparison point."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The systematic testing of all Internal (Mi) × External (Me) model pairings (9 configurations per problem source, across 3 problem sources = 27 configurations in Table 3) functions as an ablation that isolates the contribution of the internal model vs. external model vs. problem source."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses three primary metrics (Success Rate, Rediscovery, Novel & Valid), supplemented by semantic coherence (cosine similarity, Euclidean distance), readability scores (Flesch-Kincaid), and the Generalizer deficit score with four sub-criteria."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes a 'Human-Verified Competitive Ranking' where 'the authors of this study served as human evaluators' in a head-to-head tournament with pairwise preferences, yielding ELO ratings (Table 5). Detailed case studies are in Appendix C."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The entire evaluation dataset consists of ICLR 2025 papers with submission deadlines after the models' knowledge cutoffs, creating a natural temporal separation: 'all models used in our experiments have knowledge cutoffs that predate the ICLR 2025 submission deadline' (Section 4.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 provides per-tier breakdowns (Oral, Spotlight, Poster) of Success Rate, Rediscovery Rate, and Novel & Valid Score. The qualitative analysis in Section 5 and Table 8 provides per-cluster breakdowns of solution archetypes."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where models fail at strict rediscovery (τ=5 scores plummet to 15-20% from 75-84% at τ=4, Table 3). The ELO tournament includes cases where LLM solutions lose to human abstracts (Appendix C, Examples 2 and 7)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that strict rediscovery is 'exceptionally rare' (15-20% at τ=5), that Mistral-24B substantially underperforms larger models, and acknowledges 'problem-solving ability remains fragile and highly sensitive to framing' (Abstract)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims LLMs can 'rediscover feasible solutions and occasionally propose creative alternatives' — supported by Success Rate data in Table 3 (up to 74% strict). The claim that 'problem-solving ability remains fragile and highly sensitive to framing' is supported by the large gap between τ=4 and τ=5 thresholds and the dependence on problem source."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The primary causal claim is that the internal model's capability determines solution quality. This is supported by the systematic combinatorial design (all Mi × Me pairings) that controls for the external model and problem source, effectively isolating the internal model variable. The ablation-style design is adequate for this claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to assess 'AI-Generated Approaches to Research Problems' broadly, but the study tests only on AI/ML papers from ICLR 2025. The conclusion acknowledges 'Our work is primarily focused on the AI domain' but the title and abstract ('research problems') suggest broader generality. The paper also uses only 3 model families."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for its results. For instance, the high Success Rates could reflect LLM-as-a-judge bias (the judge is one of the tested models — GPT-OSS-120B) rather than genuine problem-solving. The paper validates with a second judge (Qwen-235B) but does not discuss whether both judges share systematic biases, or whether 'successful' solutions are just plausible-sounding text."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses pseudonymized model names (GPT-OSS-120B, Qwen-235B, Mistral-24B) without specifying exact model versions, snapshot dates, or API version identifiers. No specific version strings (e.g., 'gpt-4-0613') are provided. The use of pseudonyms makes it impossible to determine what was actually tested."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides the exact prompts for the Generalizer agent, Solver agent, Generalizer critic, and Solution critic, including system roles and user prompts with placeholder variables clearly marked."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports MaxInternalAttempts=20 and MaxExternalAttempts=20 but does not specify LLM inference parameters such as temperature, top-p, max tokens, or other sampling settings for any of the three models used."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The dual-loop iterative refinement scaffold (internal + external critique) is described in detail in Section 3.2 and Algorithm 1, including the roles of Mi and Me, accept/reject signals, and maximum iteration counts."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 and Table 1 describe the data curation: 1,214 papers from ICLR 2025 stratified by acceptance tier, with the stratification ratios compared against the full ICLR corpus. The oversampling of Oral and Spotlight papers is explained."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion (Section 6) mentions two limitations in a single sentence: 'Our work is primarily focused on the AI domain, and the LLM-as-a-judge paradigm carries inherent biases.' This does not constitute substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The paper's only acknowledgment of threats is the brief mention of domain limitation and judge bias in the conclusion. No specific threats are analyzed, such as the potential for the judge model (GPT-OSS-120B) to favor its own outputs, or the ELO tournament being evaluated by paper authors rather than independent evaluators."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Beyond stating the work is 'primarily focused on the AI domain,' the paper does not explicitly state what the results do NOT show. It does not bound claims about reasoning vs. recall, does not state that the results cannot speak to whether models truly 'understand' science, and does not qualify the ELO results given the small number of human comparisons (42 matches total)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the curated dataset of 1,214 papers, the generated problem statements, the generated solutions, nor the LLM judge scores are made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data collection: papers curated from ICLR 2025 conference submissions, drawing from the ICLR Dataset (González-Márquez & Kobak, 2024), stratified by acceptance tier (Oral, Spotlight, Poster), with excluded tiers (Rejected, Withdrawn) noted."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation in the ELO tournament was conducted by 'the authors of this study.' No description of how evaluators were selected (beyond being the authors), their qualifications, or potential biases from self-evaluation is provided."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented in Sections 3 and 4: abstract → problem extraction (Generalizer) → solution generation (Solver) → LLM-as-a-judge evaluation. Algorithm 1 formalizes the process. Table 1 shows input/output counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or sponsors are acknowledged anywhere in the paper. The authors are affiliated with ServiceNow Research and academic institutions, but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the first page, including LIVIA/ÉTS Montréal, Mila, HEC Montréal, ServiceNow Research, Canada CIFAR AI Chair, Université de Montréal, and Polytechnique Montréal."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Two authors are affiliated with ServiceNow Research, which has commercial interests in AI capabilities, yet this potential conflict is not addressed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Authors from ServiceNow Research may have commercial interests related to the findings, but this is not disclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'all models used in our experiments have knowledge cutoffs that predate the ICLR 2025 submission deadline' (Section 4.1) but does not specify the actual training cutoff dates for any of the three models. The exact cutoff dates are needed to assess contamination risk."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly discusses this: 'To prevent data leakage and ensure our evaluation tests reasoning rather than retrieval, all models used in our experiments have knowledge cutoffs that predate the ICLR 2025 submission deadline.' The use of post-cutoff papers is the contamination mitigation strategy."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper's experimental design specifically addresses contamination by using ICLR 2025 submissions that postdate model training cutoffs. Additionally, the problem extraction phase strips solution details from abstracts, creating a second layer of contamination protection."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The human evaluation component is a small expert annotation task by the paper authors (ELO tournament), not a formal human subjects study requiring pre-registration."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The human evaluation involved only the paper's authors as evaluators, not external human subjects. No IRB review would be required."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The evaluators were the paper's authors, not recruited study participants. Demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No external participants were recruited; the paper's authors served as evaluators."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects experiment was conducted. The ELO tournament used randomized and anonymized presentation of solutions, but this is not a human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study. The ELO tournament mentions solutions were 'presented in a randomized and anonymized fashion,' but this is part of the annotation protocol, not a human subjects blinding design."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No external human participants were recruited, so attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or per-example costs are reported. The framework uses up to 20 internal × 20 external iterations per paper across 1,214 papers and 27 configurations, representing potentially massive API costs, but these are not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, API spend, or wall-clock time is reported. Given the scale (1,214 papers × 27 configurations × up to 400 iterations each), the total cost could be substantial but is not disclosed."
       }
     }
   },
   "claims": [
     {
       "claim": "The internal model's capability is the single most predictive factor of success in scientific problem-solving.",
       "evidence": "Table 3 shows GPT-OSS-120B as internal agent achieves 74.05% strict Success Rate vs. 43.82% for Qwen-235B and 34.60% for Mistral-24B. This pattern holds across all 27 configurations. Table 5 ELO ratings confirm GPT-OSS-120B self-play (1119) far exceeds other configurations (939, 927, 828).",
       "supported": "strong"
     },
     {
       "claim": "LLMs rarely achieve perfect rediscovery of human solutions but often find novel, valid alternatives.",
       "evidence": "Table 3 shows Rediscovery rates drop from 75-84% (τ=4) to 15-20% (τ=5), while Novel & Valid remains relatively stable (e.g., 59.39% at strict threshold for top configuration). Section 5 states 'perfect rediscovery is exceptionally rare.'",
       "supported": "strong"
     },
     {
       "claim": "LLM problem-solving ability is not significantly impacted by the perceived quality or difficulty of the research paper.",
       "evidence": "Figure 3 shows Success Rates of 69.0% (Oral), 77.8% (Spotlight), 72.5% (Poster) for the top agent, described as a 'counter-intuitive result' (Section 5). However, the differences across tiers are modest and not tested for significance.",
       "supported": "moderate"
     },
     {
       "claim": "The findings are robust to evaluator choice.",
       "evidence": "Table 4 replicates results using Qwen3-235B as judge instead of GPT-OSS-120B, showing the same patterns (Rediscovery drop from τ=4 to τ=5, GPT-OSS-120B dominance). However, both judges are LLMs and may share systematic biases.",
       "supported": "moderate"
     },
     {
       "claim": "These findings provide the first large-scale evidence on whether LLMs can act as autonomous scientific problem-solvers.",
       "evidence": "The study covers 1,214 ICLR papers, which is larger than prior work cited. However, 'first large-scale' is hard to verify without exhaustive literature review, and the scope is limited to AI/ML papers.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AInstein evaluates LLM scientific problem-solving on 1,214 ICLR 2025 papers using iterative critique loops for problem extraction and solution generation. The key finding is that the internal model's capability is the dominant factor in solution quality, with GPT-OSS-120B achieving 74% strict Success Rate vs. 44% for Qwen-235B. Perfect rediscovery of human solutions is rare (15-20% at strict threshold), but models frequently generate novel, valid alternatives (59% for the best configuration). Performance is surprisingly stable across paper quality tiers (Oral, Spotlight, Poster).",
   "red_flags": [
     {
       "flag": "LLM-as-a-judge uses same model being evaluated",
       "detail": "GPT-OSS-120B serves as both the primary LLM judge and one of the models being evaluated. This creates a potential self-preferencing bias — the judge may systematically favor outputs with stylistic or structural properties it tends to produce. While validation with Qwen-235B as an alternative judge partially addresses this, both are LLMs and may share systematic biases toward plausible-sounding but incorrect solutions."
     },
     {
       "flag": "Pseudonymized model names obscure reproducibility",
       "detail": "The paper uses pseudonyms (GPT-OSS-120B, Qwen-235B, Mistral-24B) without revealing the actual model identifiers or versions. This makes the experiments impossible to replicate and prevents readers from assessing whether the models' training data truly predates ICLR 2025 submissions."
     },
     {
       "flag": "Authors served as own human evaluators",
       "detail": "The ELO tournament was conducted by the paper's authors, who may have biases toward their own framework's outputs or toward the larger model. The evaluation was done in 'randomized and anonymized fashion,' but the small number of matches (42 total, per Table 5 win/loss counts) and lack of inter-annotator agreement metrics limit the strength of this validation."
     },
     {
       "flag": "No cost reporting despite massive scale",
       "detail": "The experiment tests 27 model configurations across 1,214 papers with up to 400 LLM calls per paper (20 internal × 20 external iterations), plus LLM-as-a-judge scoring. The total API cost could be very substantial but is never reported, making it impossible to assess the method's practicality."
     },
     {
       "flag": "Minimal limitations discussion",
       "detail": "A study making claims about 'genuine reasoning' vs. 'sophisticated recall' in LLMs provides only two sentences of limitations in the conclusion. The paper does not discuss whether 'successful' solutions are truly novel reasoning or whether LLMs recombine memorized techniques in plausible-sounding ways — which is the central question the paper claims to address."
     },
     {
       "flag": "No artifacts released",
       "detail": "Despite proposing a reusable evaluation framework, the paper releases no code, data, problem statements, generated solutions, or judge scores. This prevents any independent verification of the results."
     }
   ],
   "cited_papers": [
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Foundational work on iterative LLM self-refinement, directly used in AInstein's critique loop design."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath"],
       "year": 2023,
       "relevance": "Agentic LLM framework using verbal feedback for iterative improvement, closely related to AInstein's agent architecture."
     },
     {
       "title": "G-Eval: NLG Evaluation Using GPT-4 with Better Human Alignment",
       "authors": ["Yang Liu", "Dan Iter", "Yichong Xu"],
       "year": 2023,
       "relevance": "LLM-as-a-judge methodology used as the basis for AInstein's evaluation paradigm."
     },
     {
       "title": "Curie: Evaluating LLMs on Multitask Scientific Long Context Understanding and Reasoning",
       "authors": ["Hao Cui", "Zahra Shamsi"],
       "year": 2025,
       "arxiv_id": "2503.13517",
       "relevance": "Benchmark for evaluating LLM scientific reasoning capabilities, directly relevant to the survey scope."
     },
     {
       "title": "SciBench: Evaluating College-Level Scientific Problem-Solving Abilities of Large Language Models",
       "authors": ["Xiaoxuan Wang", "Ziniu Hu", "Pan Lu"],
       "year": 2024,
       "arxiv_id": "2307.10635",
       "relevance": "Benchmark evaluation of LLM scientific problem-solving, closely related to AInstein's goals."
     },
     {
       "title": "Towards Large Reasoning Models: A Survey of Reinforced Reasoning with Large Language Models",
       "authors": ["Fengli Xu", "Qianyue Hao"],
       "year": 2025,
       "arxiv_id": "2501.09686",
       "relevance": "Survey of LLM reasoning capabilities, directly relevant to the survey's coverage of LLM capability evaluation."
     },
     {
       "title": "Knowledge Augmented Complex Problem Solving with Large Language Models: A Survey",
       "authors": ["Da Zheng", "Lun Du", "Junwei Su"],
       "year": 2025,
       "arxiv_id": "2505.03418",
       "relevance": "Survey on LLM problem-solving with knowledge augmentation, relevant to understanding LLM capability boundaries."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "relevance": "Critical analysis of LLM emergent abilities and measurement artifacts, relevant to methodology quality in AI evaluations."
     },
     {
       "title": "Mathematical Discoveries from Program Search with Large Language Models",
       "authors": ["Bernardino Romera-Paredes"],
       "year": 2024,
       "relevance": "Demonstrates LLMs making novel mathematical discoveries, directly relevant to the AI scientific reasoning literature."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational work on in-context learning capabilities of LLMs, relevant to the survey's scope on LLM capabilities."
     },
     {
       "title": "Advancing the Scientific Method with Large Language Models: From Hypothesis to Discovery",
       "authors": ["Yanbo Zhang", "Sumeer A. Khan"],
       "year": 2025,
       "arxiv_id": "2505.16477",
       "relevance": "Directly relevant to AI-driven scientific discovery, assessing LLM capability for hypothesis generation."
     },
     {
       "title": "SciMON: Scientific Inspiration Machines Optimized for Novelty",
       "authors": ["Qingyun Wang", "Doug Downey", "Heng Ji", "Tom Hope"],
       "year": 2024,
       "relevance": "System for generating novel scientific ideas, directly comparable to AInstein's goal of LLM-generated research approaches."
     }
   ]
 }