ai-research-survey

scan.json (25827B)

---

 {
   "paper": {
     "title": "The Ramón Llull's Thinking Machine for Automated Ideation",
     "authors": [
       "Xinran Zhao",
       "Boyuan Zheng",
       "Chenglei Si",
       "Haofei Yu",
       "Ken Ziyu Liu",
       "Runlong Zhou",
       "Ruochen Li",
       "Tong Chen",
       "Xiang Li",
       "Yiming Zhang",
       "Tongshuang Wu"
     ],
     "year": 2025,
     "venue": "COLM 2025 (workshop paper)",
     "arxiv_id": "2508.19200",
     "doi": "10.48550/arXiv.2508.19200"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'We will open-source our code, data, and generated research ideas' (Section 1) — future tense constitutes a promise, not an actual release. The GitHub URL is provided but 'will' indicates it was not released at time of writing."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses papers collected from Paper Copilot (publicly available) from ICLR 24, COLM 24, COLT 24, ACL 24/23/22, which are public conference proceedings. The element extraction data is promised for release but the source data is public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided. Only 'Scikit-Learn' is mentioned for TF-IDF/t-SNE (Appendix A.6)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The pipeline is described conceptually but there are no runnable commands or scripts documented."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 reports only point estimates for diversity, similarity, and relevance metrics with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares four ideation methods (Table 4) and claims differences in diversity/similarity/relevance but uses no statistical tests to support these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw metric values are reported but no effect sizes, percentage improvements with baseline context, or magnitude-of-difference measures are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "100 ideas per method are used for comparison (Table 4), and 2000 papers sampled from ICLR 24, but no justification is given for why these sample sizes are adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experimental results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against Si et al. (2024) and Yu et al. (2024) as baselines for ideation quality (Table 4, Section 4.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Both baselines — Si et al. (2024) and Yu et al. (2024) — are recent and represent current state-of-the-art in LLM-based ideation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Two variants of the thinking machine are compared: Ramón Llull (Top) using most-visited elements and Ramón Llull (Random) using random sampling, which tests the effect of element selection strategy (Table 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: diversity (distinct-1), similarity (Jaccard), and relevance (BLEU) (Section 4.2, Table 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated ideas is performed. The paper explicitly acknowledges this gap: 'Rigorous idea quality evaluation may involve extensive expert annotation' (footnote 7) and discusses it in Limitations (A.1)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "ACL 2025 main paper titles (released after June 2025) are used as the reference set for similarity and relevance, which post-dates the models used (Gemini 1.5 Pro released Feb 2024), providing a temporal hold-out (Section 4.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per conference (Table 2, Table 5, Figure 2) and per element type (Theme, Domain, Method), with qualitative comparisons in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 discusses what is NOT covered by the A+B+C framework (perturbation, 4th axis, negation), and the bijective coverage analysis (Table 5) shows only 16.4% reconstructibility, which is discussed as a limitation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The low reconstructibility rate (16.4%, Table 5) is reported and discussed honestly. The paper also reports that Ramón Llull (Random) has much lower relevance (0.05) compared to prior methods (0.28, 0.18)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims ideas are 'diverse, relevant, and grounded in current literature.' Table 4 shows diversity and relevance metrics; the element mining from conference papers provides grounding. The claims are hedged appropriately."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implicitly claims that conditioning on explicit concept combinations 'helps' diversity (Section 1) and that random sampling 'leads to' higher diversity (Section 4.2). These are causal claims from observational comparisons without controlling for confounds (different generation processes, different models for rewriting)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its claims to ML/NLP conferences and notes the framework's limitations. Footnote 2 explicitly states diversity and coverage 'do not necessarily suggest the novelty and utility of the ideas.' Section 4.4 discusses what isn't covered."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its metric results. For instance, the higher diversity of Ramón Llull (Random) could be due to random word combinations rather than genuinely diverse ideas. No confounds are discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to distinguish its metrics (diversity, similarity, relevance) from actual idea quality. Footnote 7 states 'The results from our metrics do not suggest the superior quality of any method.' Appendix A.1 discusses the gap between lexical diversity and scientific merit."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Gemini 2.0 Flash', 'Gemini-1.5 Pro', and 'Claude 3.7' without specific version IDs or snapshot dates. These are marketing names only."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in Appendix A.6 for element extraction, element merging, idea rewriting, and in A.8 for decomposition and reconstruction evaluation. These are complete, usable prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.6 states: 'max output token to be 8192, temperature to be 0.7, top p to be 0.7, and top k to be 50.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The pipeline consists of direct LLM API calls for extraction, merging, and rewriting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The element mining pipeline is described: papers collected from Paper Copilot, processed by Gemini for element extraction, then merged based on semantic similarity. Table 2 provides counts at each stage. Section 3.2 describes the pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 contains a dedicated limitations section with two substantive subsections: 'Evaluating idea quality and novelty' and 'Organizing the elements.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations discuss specific issues: lexically unique ideas may be conceptually trivial, high relevance may indicate incrementality rather than novelty, LLM-as-judge may favor well-phrased but shallow ideas, and flat element lists lose hierarchical structure (Appendix A.1)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 explicitly identifies dimensions NOT covered by the framework (perturbation, 4th axis, negation). Footnote 2 states what the metrics do NOT measure. The paper states 'the proposed Ram´on Llull thinking machine is NOT intended to' with specific exclusions (Section 5)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The extracted elements, templates, and generated ideas are promised for future release but not available at time of writing. Only aggregated statistics (Table 2) are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Papers were collected from Paper Copilot for six specific conferences. 2000 papers randomly sampled from ICLR 24. Processing used Gemini 2.0 Flash with documented prompts (Section 3.2, Appendix A.6)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied. The PhD student annotators in the pilot (Section 3.2) are contributors, not study subjects. Data comes from public conference proceedings."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from paper collection → element extraction → merging → template combination → LLM rewriting is documented with prompts (A.6), statistics (Table 2), and processing details."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists ONR Award N000142312840, OpenAI Research Credit program, Amazon AI Research Gift Fund, and Gemma Academic Program GCP Credit Award."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: CMU, OSU, Stanford, UIUC, UT Dallas, UW."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The work is funded by OpenAI Research Credits and uses Gemini models. Both OpenAI and Google have financial interests in LLM ideation capabilities. The paper does not acknowledge this potential conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes Gemini 1.5 Pro was released Feb 2024 to justify using ACL 2025 as reference, but does not state training cutoff dates for Gemini 2.0 Flash or Claude 3.7, which are used for element extraction and idea generation."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 explicitly addresses this: 'the accepted papers are released after June 2025. To reduce the chance of LLMs seeing the paper titles in their training data (Gemini 1.5 Pro was released Feb 2024).' Temporal separation is used as a mitigation."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While temporal separation is used for the ACL 2025 reference set, the element extraction from ICLR/ACL/COLT/COLM papers (2022-2024) is done with models that likely trained on these papers. This contamination of the element mining step is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline processes 7,483 papers through multiple LLM calls (extraction, merging, rewriting, evaluation) but no API costs, token counts, or wall-clock times are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No information about total API spend, compute resources, or processing time is provided despite extensive LLM usage across the pipeline."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. LLM generation with temperature 0.7 is inherently stochastic but no multiple-run analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "It is unclear how many times the LLM generation was run. Results appear to be from single runs with no repetition stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.7, top_p 0.7, top_k 50 are used but no justification or search process for these settings is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of top-20 elements per disk, the basic A+B+C template for visualization, and the sampling of 100 ideas for comparison are not justified."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their own system against baselines from Si et al. (2024) and Yu et al. (2024) without acknowledging potential bias in how the comparison is set up (e.g., choice of metrics, reference set)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different methods likely have very different compute costs (e.g., Ramón Llull with simple template fills vs. multi-agent simulation in Yu et al.) but no compute-performance comparison is made."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses construct validity extensively: footnote 7 notes metrics don't measure quality, Appendix A.1 discusses how diversity/relevance can be misleading, and Section 4.4 discusses what the framework cannot capture."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved in the evaluation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses ACL 2025 titles (released June 2025) as reference, explicitly noting Gemini 1.5 Pro was released Feb 2024 to create temporal separation (Section 4.2)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The element extraction uses LLMs that likely have the source conference papers in their training data. When Gemini extracts elements from ICLR 2024 papers, it may leverage memorized content rather than extracting from the provided text. This is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Elements are mined from the same conferences used in evaluation (e.g., elements from ACL 2024 compared against ACL 2025). Papers within conferences share topics, authors, and citation networks, creating non-independence that is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. Only temporal separation (using post-training reference set) is used, which is a prevention strategy for the reference set but not for the element mining step."
       }
     }
   },
   "claims": [
     {
       "claim": "Near-universal decomposability (99.5%): almost all research papers can be decomposed into the A+B+C (Theme, Domain, Method) framework.",
       "evidence": "Table 5 shows 99.5% decomposition rate across 7,483 papers from six conferences (Section 4.3).",
       "supported": "weak"
     },
     {
       "claim": "Reconstructibility is limited (16.4%): only a fraction of papers can be faithfully reconstructed from their elements.",
       "evidence": "Table 5 shows 16.4% reconstruction rate using ≥30% Jaccard similarity threshold (Section 4.3).",
       "supported": "moderate"
     },
     {
       "claim": "Ramón Llull (Random) achieves the highest diversity (0.41 distinct-1) among compared methods.",
       "evidence": "Table 4 compares four methods: Si et al. (0.29), Yu et al. (0.29), Ramón Llull Top (0.21), Ramón Llull Random (0.41).",
       "supported": "weak"
     },
     {
       "claim": "There is a trade-off between diversity and similarity/relevance in the ideation methods.",
       "evidence": "Table 4 shows Ramón Llull (Random) has highest diversity (0.41) but lowest relevance (0.05), while Ramón Llull (Top) has highest similarity (0.26) but lowest diversity (0.21).",
       "supported": "moderate"
     },
     {
       "claim": "Different conferences occupy different regions of the idea space, with COLT being most standalone and COLM lying at the intersection.",
       "evidence": "Figure 2 shows t-SNE density heatmaps of generated ideas by conference, with qualitative analysis in Section 4.1.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "The paper proposes a combinatorial ideation framework based on Ramón Llull's thinking machine, using Theme/Domain/Method elements mined from 7,483 conference papers. The framework achieves 99.5% decomposability but only 16.4% reconstructibility, suggesting research ideation is partially but not fully combinatorial. Random element sampling produces more diverse but less relevant ideas than top-element enumeration, revealing a diversity-relevance trade-off. The approach serves as a lightweight baseline for LLM ideation research.",
   "red_flags": [
     {
       "flag": "Decomposability metric is trivially high",
       "detail": "99.5% decomposability is evaluated by asking Gemini whether it can assign theme/domain/method labels to paper titles. Given these are very broad categories, nearly anything can be tagged with some theme/domain/method. This near-100% rate reveals more about the breadth of the categories than about the framework's validity."
     },
     {
       "flag": "No statistical rigor in comparisons",
       "detail": "Table 4 compares four ideation methods on three metrics with no error bars, no significance tests, and no repeated runs. With stochastic LLM generation (temperature 0.7), single-run comparisons are unreliable."
     },
     {
       "flag": "Evaluation metrics may not measure idea quality",
       "detail": "The paper acknowledges this (footnote 7, Appendix A.1), but the entire experimental section is built on these metrics. Distinct-1 diversity, Jaccard similarity, and BLEU relevance are lexical metrics that say nothing about scientific merit, feasibility, or novelty of generated ideas."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "Si et al. (2024) ideas were carefully filtered for quality (93 ideas on 7 topics), while Yu et al. (2024) used multi-agent discussion. The Ramón Llull pipeline uses simple template filling. Comparing these very different generation processes on lexical diversity/similarity metrics without controlling for generation effort or filtering is misleading."
     }
   ],
   "cited_papers": [
     {
       "title": "Can LLMs Generate Novel Research Ideas? A Large-Scale Human Study with 100+ NLP Researchers",
       "authors": ["Chenglei Si", "Diyi Yang", "Tatsunori Hashimoto"],
       "year": 2024,
       "relevance": "Directly relevant as a major baseline for LLM-based research ideation with human evaluation."
     },
     {
       "title": "The Ideation-Execution Gap: Execution Outcomes of LLM-Generated versus Human Research Ideas",
       "authors": ["Chenglei Si", "Tatsunori Hashimoto", "Diyi Yang"],
       "year": 2025,
       "arxiv_id": "2506.20803",
       "relevance": "Studies execution outcomes of LLM-generated research ideas, directly relevant to evaluating AI-assisted ideation."
     },
     {
       "title": "ResearchTown: Simulator of Human Research Community",
       "authors": ["Haofei Yu"],
       "year": 2024,
       "relevance": "Multi-agent community simulation for research ideation, a baseline compared in this paper."
     },
     {
       "title": "ResearchAgent: Iterative Research Idea Generation over Scientific Literature with Large Language Models",
       "authors": ["Jinheon Baek", "Sujay Kumar Jauhar", "Silviu Cucerzan", "Sung Ju Hwang"],
       "year": 2025,
       "relevance": "Iterative LLM-based ideation with knowledge augmentation, part of the automatic ideation landscape."
     },
     {
       "title": "CodeScientist: End-to-End Semi-Automated Scientific Discovery with Code-Based Experimentation",
       "authors": ["Peter Jansen"],
       "year": 2025,
       "arxiv_id": "2503.22708",
       "relevance": "End-to-end automated scientific discovery system, related to AI-assisted research workflows."
     },
     {
       "title": "Scideator: Human-LLM Scientific Idea Generation Grounded in Research-Paper Facet Recombination",
       "authors": ["Marissa Radensky"],
       "year": 2024,
       "relevance": "Concurrent work on facet-based recombination for ideation, closely related approach to this paper."
     },
     {
       "title": "The Impact of Large Language Models on Scientific Discovery: A Preliminary Study Using GPT-4",
       "authors": ["Microsoft Research AI4Science", "Microsoft Quantum"],
       "year": 2023,
       "relevance": "Early study of LLM impact on scientific discovery, foundational to the AI-for-science direction."
     },
     {
       "title": "AI2 Scholar QA: Organized Literature Synthesis with Attribution",
       "authors": ["Amanpreet Singh"],
       "year": 2025,
       "arxiv_id": "2504.10861",
       "relevance": "AI-powered literature synthesis tool, relevant to automated research workflows."
     }
   ]
 }