ai-research-survey

scan.json (31172B)

---

 {
   "paper": {
     "title": "Hypothesis Generation for Materials Discovery and Design Using Goal-Driven and Constraint-Guided LLM Agents",
     "authors": [
       "Shrinidhi Kumbhar",
       "Venkatesh Mishra",
       "Kevin Coutinho",
       "Divij Handa",
       "Ashif Iquebal",
       "Chitta Baral"
     ],
     "year": 2025,
     "venue": "North American Chapter of the Association for Computational Linguistics (NAACL)",
     "arxiv_id": "2501.13299",
     "doi": "10.48550/arXiv.2501.13299"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Integrating knowledge graphs and iterative critic feedback into an LLM-based hypothesis generation framework (ACCELMAT) improves both closeness to ground truth (80% vs 70% baseline) and quality scores (89% vs 79.67%) on the authors' MATDESIGN benchmark of 50 materials science problems. The temporal design using January 2024 papers addresses contamination relative to late-2023 model training cutoffs. Human expert evaluation by 4 PhD students qualitatively paralleled the automated LLM-based evaluation rankings, though no inter-rater reliability statistics were reported.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 provides a GitHub URL: 'https://github.com/shri071/Hypothesis-Generation-for-Materials-Discovery-and-Design-Using-Goal-Driven-and-Constraint-Guided-LLM'. The abstract also states 'Data and code are available.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The MATDESIGN dataset is released alongside the code at the same GitHub repository per footnote 1."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment file, or environment setup section is mentioned in the paper. Only the LLM model names are listed."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides full prompts in the appendices but does not include step-by-step reproduction instructions, scripts to replicate experiments, or a README with commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Figure 2 and Section 7 are reported as point estimates (e.g., '70%', '80%', '89%') with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'a 6.67% improvement over the feedback-only setup' are made by comparing raw percentages without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context, e.g., 'Closeness score of 73.33%, a 3.33% improvement' over 70%, and 'Quality score of 85.67%, marking a 6% increase over the feedback-free setup' (Section 7). Absolute and relative differences are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 50 instances from 50 papers. The Limitation section acknowledges 'its size (50 papers) may not fully capture the diversity of materials science research' but provides no power analysis or formal justification for this sample size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across runs, or spread measures are reported. Results appear to be from single experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 establishes the 'Hypotheses Generation without Feedback from Critics' as a baseline. Three configurations are compared progressively (no feedback, with feedback, with KG and feedback)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 compares ACCELMAT with LLMatDesign, ChemReasoner, and SciAgents only on qualitative features (diverse materials, tool-free, etc.), not on actual performance. No external system is run on MATDESIGN for quantitative comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The three configurations progressively add components: (1) HGA + CA without feedback, (2) + iterative feedback with SA, (3) + knowledge graph. This functions as a progressive ablation study showing each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics are used: Closeness (with 3 sub-metrics: Concept Overlap, Property Overlap, Keyword Matching) and Quality (with 6 sub-metrics: Alignment, Plausibility, Innovation, Testability, Feasibility, Impact). See Section 5."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1: '42 suggestions, encompassing all three configurations, were independently evaluated by four PhD students in Materials Science. Their assessments employed the same evaluation metrics used by the automated system.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 50 MATDESIGN instances appear to be used for evaluation. No train/dev/test split is described, and it is unclear whether any instances were used during system development or prompt tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides per-criterion breakdowns for both Closeness (3 criteria) and Quality (6 criteria) across all three configurations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific failure modes for each configuration: 'Lack of Consensus,' 'Incomplete Adherence to Constraints,' 'Bias in Material and Method Selection,' and for KG: 'information retrieved from the knowledge graph... tends to focus on fundamental materials.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 'hypotheses remained focused on well-established methodologies, with limited exploration of unconventional solutions' even with feedback, and that KG information sometimes produced 'broad and generalized suggestions that lack the specificity needed for practical implementation.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about curating a novel dataset, testing LLM-based agents, and proposing a scalable evaluation metric are all supported by Sections 3, 4-6, and 5 respectively. Claims are hedged appropriately ('explore the potential,' 'aim to advance')."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims feedback and KG integration 'enhance performance.' The progressive addition of components (no feedback → feedback → KG + feedback) with the same base model (GPT-4o) constitutes controlled single-variable manipulation adequate for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Materials Discovery and Design' generally, and the abstract says the framework aims 'to advance future research in accelerating materials discovery and design with LLMs.' However, results are from 50 specific instances from journals in a single month (January 2024), with a single hypothesis generator (GPT-4o). These scope limitations are not acknowledged in the claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered for why KG+feedback outperforms other configurations. For example, the KG configuration provides more context tokens to the generator — the improvement could be due to additional context rather than structured knowledge. This confound is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures LLM-scored Closeness and Quality metrics and frames these as evaluating 'materials discovery and design.' The gap between LLM evaluation scores (a proxy) and actual scientific utility of hypotheses is not discussed. The BERTScore comparison in Appendix G actually contradicts the LLM evaluator rankings, but this discrepancy is dismissed rather than explored."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names: 'GPT-4o,' 'Claude-3.5-Sonnet,' 'Gemini-1.5-Flash,' 'OpenAI-o1-preview.' No API version identifiers, snapshot dates, or specific model IDs (e.g., 'gpt-4o-2024-05-13') are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendices C and D, including the hypothesis generator prompt (C.3), critic feedback prompt (C.4), summarizer prompt (C.5), and evaluation prompts (Appendix E). These contain actual prompt text, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for any of the LLMs used (GPT-4o, Claude-3.5-Sonnet, Gemini-1.5-Flash, o1-preview)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent framework is described in detail in Section 4: HGA, three CAs, SA, and EA roles are defined. Algorithm 1 provides pseudocode for the iterative feedback loop. The knowledge graph integration workflow is described in Section 6.3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 3 describes the dataset structure (goals, constraints, materials, methods from 50 papers) and mentions expert assistance, but does not document the full pipeline: how many papers were initially considered, what specific inclusion/exclusion criteria were used beyond 'prominent journals' and 'January 2024,' or how expert extractions were validated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitation' section is present, discussing dataset size constraints, LLM feedback reliability, hallucination risks, and dependency on human-provided constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'its size (50 papers) may not fully capture the diversity,' 'even unanimous agreement among state-of-the-art LLM-based critics does not guarantee scientific accuracy,' and 'the risk of hallucinated or flawed suggestions remains a challenge.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound the scope to specific material types, application domains, or settings where the approach may not generalize. The Limitation section discusses expansion opportunities but not explicit exclusions from claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The MATDESIGN dataset and code are released on GitHub (footnote 1), enabling independent verification of the benchmark data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection: extracted from 50 research papers published from January 2024 in prominent journals (Nature, Nature Communications, Progress in Organic Coatings), with materials science expert assistance for all extractions."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the human evaluation (Section 7.1), '42 suggestions... were independently evaluated by four PhD students in Materials Science' but no recruitment details are provided — how the 4 PhD students were selected, from which lab, or whether this introduces evaluation bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states information was extracted from 50 papers but does not document the pipeline with counts at each stage (e.g., how many journals were searched, how many papers were considered, filtering criteria at each step, or inter-annotator agreement for the expert extractions)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgement section: 'This research was supported by the Engineering Research and Development Center - Information Technology Laboratory (ERDC-ITL) under Contract No. W912HZ24C0022.' Also acknowledges ASU Research Computing resources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed with Arizona State University affiliation. They use third-party models (GPT-4o, Claude, Gemini) without being affiliated with those companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "ERDC-ITL is a government research lab with no direct financial stake in whether particular LLM-based hypothesis generation approaches outperform others. The disclaimer states 'Any opinions, findings and conclusions... do not necessarily reflect the views of the ERDC-ITL.'"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 2: 'Llama-3.1-70B (Model Card) and GPT-4o (Documentation) have knowledge cutoffs of December 2023 and October 2023 respectively.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3: 'The selection of publications from January 2024 is essential to ensure that the ground truth information... is not present in the training corpus of LLMs we use, which have a training cutoff of late 2023.' This temporal design addresses potential overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The entire benchmark design is built around contamination avoidance: MATDESIGN uses papers published after model training cutoffs. Section 2.2 and Table 2 explicitly list 'No Data Leakage' as a feature distinguishing MATDESIGN from prior benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration for the human evaluation study with 4 PhD students."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval for the human evaluation involving 4 PhD students."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The only demographic information for human evaluators is 'four PhD students in Materials Science.' No experience level, years in program, gender, or other characterization is provided."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No criteria are stated for selecting the 4 PhD student evaluators — why these students, what qualifications were required, etc."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human evaluation is an expert rating task, not an experimental study with treatment/control conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "The human evaluation is an expert rating task assessing output quality, not an experimental study where blinding would be applicable."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether all 4 PhD students completed all evaluations or if any dropout occurred."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions wanting to 'restrict the cost per instance and the runtime' (Section 7) but does not report actual API costs, tokens consumed, or wall-clock time for any configuration."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges use of 'ChatGPT enterprise version' and ASU computing resources but does not quantify total API spend, compute hours, or hardware used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or sensitivity analysis. Results appear to be from single runs of each configuration."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results represent single runs or averages."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The number of hypotheses (20), number of feedback cycles (5), and number of critics (3) appear to be fixed without justification or search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All three configurations are predefined and all results are reported; there is no cherry-picking from a larger set of configurations."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created MATDESIGN, designed ACCELMAT, and developed the evaluation metric, then evaluated their system on their benchmark with their metric. This self-evaluation bias is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The KG+feedback configuration requires significantly more API calls (knowledge graph queries, additional feedback cycles) than the no-feedback baseline, but performance is not normalized by compute budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether MATDESIGN actually measures the ability to generate viable hypotheses for materials discovery. The Closeness metric measures similarity to known solutions, which may penalize truly novel hypotheses — this construct validity issue is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The scaffold (feedback loops, KG integration) IS the variable being tested across configurations, with the same base model (GPT-4o) throughout. The scaffold is the thing being evaluated, not a confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark uses papers from January 2024, after model training cutoffs of late 2023, explicitly designed to prevent temporal leakage. Section 3 and footnote 2 discuss this."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Evaluation Agent (o1-preview) receives both generated hypotheses and ground truth for scoring. Whether o1-preview might have encountered the ground truth papers (published January 2024, potentially within o1-preview's training data given its later release) is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Whether the 50 MATDESIGN papers share structural similarities (e.g., overlapping domains, similar material types, common journal conventions) that could inflate performance estimates is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "A temporal split (January 2024 papers vs late 2023 training cutoffs) is used as a concrete decontamination method. Table 2 lists 'No Data Leakage' as a verified property of MATDESIGN."
       }
     }
   },
   "claims": [
     {
       "claim": "Integrating knowledge graphs and critic feedback (KG+feedback) achieves the best performance with 80% Closeness and 89% Quality scores.",
       "evidence": "Figure 2 and Section 7: Closeness scores of 70% (no feedback), 73.33% (feedback), 80% (KG+feedback). Quality scores of 79.67%, 85.67%, 89% respectively.",
       "supported": "moderate"
     },
     {
       "claim": "Critic feedback increases consensus among LLM critics from 11/20 to 19/20 agreed hypotheses.",
       "evidence": "Table 5 shows average hypotheses agreed by all critics: 11 (without feedback), 18 (with feedback), 19 (with KG and feedback).",
       "supported": "moderate"
     },
     {
       "claim": "The proposed LLM-based evaluation metric aligns with human expert judgments.",
       "evidence": "Section 7.1: 'the human evaluations paralleled the automated results' — configuration rankings (no feedback < feedback < KG+feedback) matched between human and automated evaluation.",
       "supported": "weak"
     },
     {
       "claim": "MATDESIGN benchmark ensures no data leakage by using papers published after model training cutoffs.",
       "evidence": "Section 3 and footnote 2: Papers from January 2024 vs model cutoffs of October/December 2023. Table 2 lists 'No Data Leakage' as verified.",
       "supported": "strong"
     },
     {
       "claim": "Closed-source models consistently outperform open-source counterparts on both Closeness and Quality metrics.",
       "evidence": "Appendix A and Figure 3: Comparison using LLaMA-3.1 70B, Gemma-2-27B, Mixtral-8x22B (open) vs GPT-4o, Claude-3.5-Sonnet, Gemini-1.5-Flash (closed).",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "LLM-as-judge without adequate validation",
       "detail": "OpenAI o1-preview serves as the primary evaluation agent, scoring hypotheses on both Closeness and Quality. The BERTScore comparison (Appendix G) contradicts the LLM evaluator rankings — KG+feedback scores lowest (50.30%) on BERTScore but highest on the LLM metric. This discrepancy is dismissed rather than investigated, undermining confidence in the evaluation."
     },
     {
       "flag": "Tiny sample size with no statistical tests",
       "detail": "All conclusions are drawn from 50 benchmark instances with no confidence intervals, significance tests, or repeated runs. Reported differences (e.g., 3.33% and 6.67% improvements) could easily be within noise for N=50."
     },
     {
       "flag": "Self-evaluation loop",
       "detail": "The authors created the benchmark (MATDESIGN), the system (ACCELMAT), and the evaluation metric, then evaluated their system on their benchmark with their metric. No independent evaluation exists."
     },
     {
       "flag": "Anecdotal human evaluation",
       "detail": "Only 4 PhD students evaluated 42 suggestions with no inter-rater reliability statistics (Kappa, ICC), no blinding information, no recruitment details. The alignment between human and automated evaluation is described qualitatively ('paralleled') without quantitative comparison."
     },
     {
       "flag": "No cost reporting despite multi-LLM system",
       "detail": "The system uses GPT-4o, Claude-3.5-Sonnet, Gemini-1.5-Flash, and o1-preview across multiple iterative cycles per instance. Total API cost is never reported, making practical feasibility impossible to assess."
     }
   ],
   "cited_papers": [
     {
       "title": "LLMatDesign: Autonomous Materials Discovery with Large Language Models",
       "authors": ["Shuyi Jia", "Chao Zhang", "Victor Fung"],
       "year": 2024,
       "arxiv_id": "2406.13163",
       "relevance": "LLM-based multi-agent framework for materials design hypothesis generation, directly comparable to ACCELMAT."
     },
     {
       "title": "ChemReasoner: Heuristic Search over a Large Language Model's Knowledge Space Using Quantum-Chemical Feedback",
       "authors": ["Henry W Sprueill", "Carl Edwards", "Khushbu Agarwal"],
       "year": 2024,
       "arxiv_id": "2402.10980",
       "relevance": "LLM agent using heuristic search and chemical feedback for catalyst discovery, a comparable domain-specific LLM agent approach."
     },
     {
       "title": "SciAgents: Automating Scientific Discovery through Multi-Agent Intelligent Graph Reasoning",
       "authors": ["Alireza Ghafarollahi", "Markus J Buehler"],
       "year": 2024,
       "arxiv_id": "2409.05556",
       "relevance": "Multi-agent LLM framework for scientific discovery using knowledge graphs, similar architecture to ACCELMAT."
     },
     {
       "title": "Empowering Biomedical Discovery with AI Agents",
       "authors": ["Shanghua Gao", "Ada Fang", "Yepeng Huang"],
       "year": 2024,
       "relevance": "Multi-agent AI framework for biomedical discovery, examines LLM agent architectures for hypothesis generation in scientific domains."
     },
     {
       "title": "What Can Large Language Models Do in Chemistry? A Comprehensive Benchmark on Eight Tasks",
       "authors": ["Taicheng Guo", "Bozhao Nan", "Zhenwen Liang"],
       "year": 2023,
       "relevance": "ChemLLMBench: LLM benchmark for chemistry tasks, one of the baselines MATDESIGN is compared against."
     },
     {
       "title": "MaScQA: A Question Answering Dataset for Investigating Materials Science Knowledge of Large Language Models",
       "authors": ["Mohd Zaki", "NM Krishnan"],
       "year": 2023,
       "arxiv_id": "2308.09115",
       "relevance": "Materials science QA benchmark for LLMs, predecessor benchmark that MATDESIGN aims to improve upon."
     },
     {
       "title": "Honeycomb: A Flexible LLM-based Agent System for Materials Science",
       "authors": ["Huan Zhang", "Yu Song", "Ziyu Hou"],
       "year": 2024,
       "arxiv_id": "2409.00135",
       "relevance": "LLM-based agent system for materials science tasks, directly relevant to agentic AI for scientific discovery."
     },
     {
       "title": "MatExpert: Decomposing Materials Discovery by Mimicking Human Experts",
       "authors": ["Qianggang Ding", "Santiago Miret", "Bang Liu"],
       "year": 2024,
       "arxiv_id": "2410.21317",
       "relevance": "LLM-based materials discovery framework that decomposes tasks mimicking human expert workflows."
     },
     {
       "title": "MOOSE-Chem: Large Language Models for Rediscovering Unseen Chemistry Scientific Hypotheses",
       "authors": ["Zonglin Yang", "Wanhao Liu", "Ben Gao"],
       "year": 2024,
       "arxiv_id": "2410.07076",
       "relevance": "LLM framework for rediscovering chemistry hypotheses using literature and mutation algorithms."
     },
     {
       "title": "Structured Chemistry Reasoning with Large Language Models",
       "authors": ["Siru Ouyang", "Zhuosheng Zhang", "Bing Yan"],
       "year": 2023,
       "arxiv_id": "2311.09656",
       "relevance": "LLM-based structured reasoning for chemistry tasks, relevant to how LLMs handle scientific reasoning."
     },
     {
       "title": "Galactica: A Large Language Model for Science",
       "authors": ["Ross Taylor", "Marcin Kardas", "Guillem Cucurull"],
       "year": 2022,
       "arxiv_id": "2211.09085",
       "relevance": "Large language model specifically trained for scientific text, relevant to LLM capabilities in scientific domains."
     },
     {
       "title": "OpenAI o1 System Card",
       "authors": ["Aaron Jaech", "Adam Kalai", "Adam Lerer"],
       "year": 2024,
       "arxiv_id": "2412.16720",
       "relevance": "System card for o1-preview, the model used as the evaluation agent in this study."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The framework requires multiple LLM API subscriptions and a knowledge graph; the paper acknowledges outputs aren't ready for immediate practical application by materials scientists."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Results confirm the expected pattern that adding feedback and structured knowledge improves LLM output quality."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns are raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict in the findings."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released on GitHub but requires multiple API keys and domain expertise to run meaningfully."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses well-known models (GPT-4o, Claude, Gemini) but is from an academic lab (Arizona State University), not a major AI company."
     }
   }
 }