ai-research-survey

scan.json (21037B)

---

 {
   "paper": {
     "title": "Crystalyse: a multi-tool agent for materials design",
     "authors": ["Ryan Nduma", "Hyunsoo Park", "Aron Walsh"],
     "year": 2025,
     "venue": "",
     "doi": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/ryannduma/CrystaLyse.AI, also available on PyPI."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The TRINITY Gold dataset compositions are described as available in the code repository. Materials Project data is publicly accessible via API. ICSD data requires subscription but is a standard database."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is mentioned in the paper. The paper references PyPI installation but does not list library versions or dependency details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The CLI commands (crystalyse discover, crystalyse) are mentioned but no detailed protocol for reproducing the experimental results is given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The TRINITY benchmark reports 'mean ± 1 SD (n=3 runs)' for performance metrics across model-format pairs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares SMACT vs GPT-4o vs Gemini on composition validity but uses no statistical significance tests to support claims of difference."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context, e.g., 'GPT-4o improved from 64.5% to 89.4% (+25 percentage points)' and adversarial pass rate '86% from a 57% baseline'."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 2,087 compositions were chosen for TRINITY Gold, why 28 adversarial prompts were used for evaluation, or why 3 replicates were run."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation reported across 3 runs for TRINITY benchmark: 'mean ± s.d.' notation used in confusion matrices and performance metrics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "SMACT (domain tool) compared against GPT-4o and Gemini 2.0 Flash on composition validity. Adversarial testing uses a v0 baseline prompt at 57% pass rate."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o (2024-05-13) and Gemini 2.0-Flash are contemporary models used as baselines for composition validity."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The prompt engineering progression (v0 through v3) serves as an ablation showing the contribution of safety, provenance, and robustness components. The three operational modes (creative/adaptive/rigorous) are also compared."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Accuracy, precision, recall, false positive counts, and confusion matrices are reported for the TRINITY benchmark. Pass rates across multiple adversarial dimensions are also reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the agent's outputs was conducted. The materials design tasks are evaluated by the authors but no structured human evaluation protocol is described."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The adversarial testing used 28 prompts for iterative refinement and a separate complete 70-prompt suite for independent validation. The TRINITY Gold dataset is a constructed benchmark not used for system development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Performance broken down by adversarial dimension (hallucination, safety, legitimate use, sustainability, robustness) and by composition order (binary through senary) and format type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Detailed failure analysis: disguised toxic requests (25% failure rate), high-energy materials ambiguity (25% failure), render gate false positives, LLM format sensitivity. Section 'Agent robustness and failure analysis' dedicated to this."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Reported that v1 prompt over-corrected (0% robustness), that render gate had false positives, that disguised toxic requests bypass safety 25% of the time, and that LLMs show precision collapse on composition tasks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about provenance enforcement eliminating hallucinations (supported by shadow validation), 86% adversarial pass rate from 57% baseline (supported by Figure 4), and open-source release (GitHub link provided) are all supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about provenance enforcement eliminating hallucinations are supported by the prompt version ablation (v0-v3) showing controlled addition of components. The shadow validation A/B testing provides causal evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes Crystalyse as a 'prototype' initially focused on 'inorganic crystals', acknowledges limitations in property coverage (energy only, not electronic/optical/magnetic), and notes scope is bounded by accessible databases."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses whether LLM format sensitivity is due to tokenisation vs training corpus bias, investigating and ruling out tokenisation as the primary explanation (Supplementary Section S4). Limitations section discusses alternative architectural approaches."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model snapshots provided: 'o3 (snapshot o3-2025-04-16)', 'o4-mini (snapshot o4-mini-2025-04-16)', 'GPT-4o (gpt-4o-2024-05-13)', 'Gemini 2.0-Flash (gemini-2.0-flash-exp-01-21)'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "System prompts are described conceptually (v0-v3 evolution) but the actual prompt text is not provided in the paper. The paper says prompts are in supplementary sections but only describes their intent, not full text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature 0 stated for TRINITY benchmark. 'Default reasoning effort settings' stated for o3/o4-mini. Timeouts specified (60s SMACT, 300s Chemeleon, 600s MACE). Fuzzy matching tolerance ±0.001."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed description of the agentic architecture: three-layer hierarchy (orchestration, execution, validation), MCP tool integration, render gate, provenance system, clarification engine, mode selection, memory architecture. Figures 1-2 illustrate the workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "TRINITY Gold dataset construction documented: 1,500 ICSD positives and 587 MP negatives, deduplication on reduced formula, charge-neutral filtering, valid elements (Z≤118), no partial occupancies, stratified by composition order with counts (475/534/440/335/303)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Discussion section contains substantive limitations discussion: 'Limitations in our approach reflect the current toolset and model capabilities' with specific points about discovery scope, property coverage, and architectural constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: ML force field provides only point estimates without uncertainty, discovery scope bounded by databases, electronic/optical/magnetic properties not covered, disguised toxic requests bypass safety 25% of the time, render gate false positives from pattern-matching limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'initially focused on inorganic crystals', energy evaluations only (not electronic/optical/magnetic), bounded by accessible databases, and three specific principles about what was not shown."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Data stated as available in the code repository. TRINITY Gold dataset compositions and source identifiers (ICSD/MP IDs) retained for reproducibility."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "TRINITY Gold construction described in detail: 1,500 from ICSD release 2024.1, 587 from Materials Project January 2025 snapshot with Ehull > 0.5 eV/atom, tagged theoretical/never observed. Inclusion/exclusion criteria specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard computational materials databases (ICSD, Materials Project)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: source selection, deduplication on reduced formula, charge-neutral filtering, element validation, stratified sampling by composition order with counts at each stage, five encoding formats generated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding disclosed in Acknowledgements: EPSRC project EP/X037754/1, AIchemy hub EPSRC grants EP/Y028775/1 and EP/Y028759/1, including a summer studentship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors affiliated with Department of Materials, Imperial College London. The system uses OpenAI models but authors have no stated affiliation with OpenAI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "EPSRC is a UK government research council with no financial stake in the performance of Crystalyse or any particular AI model."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement found in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates GPT-4o and Gemini on the TRINITY benchmark but does not state the training data cutoff dates for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ICSD compositions or Materials Project data appeared in GPT-4o or Gemini training data. The format sensitivity finding hints at training corpus effects but contamination is not directly addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TRINITY Gold uses ICSD and Materials Project data, both publicly available before model training cutoffs. The paper discusses 'training corpus bias' but does not address whether models memorized specific compositions from training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Wall-clock times reported per mode: Creative 65-91s, Adaptive 172s, Rigorous 199-279s. Individual tool costs stated: SMACT <10ms, MACE 1-2s single-point up to ~1hr for MD. Shadow validation overhead 8.3%."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend or GPU hours reported. Hardware described as 'consumer laptops to workstations' but specific hardware not quantified. No total computational budget for the study."
       }
     }
   },
   "claims": [
     {
       "claim": "Provenance enforcement eliminated material-property hallucinations, with adversarial pass rate reaching 86% from a 57% baseline.",
       "evidence": "Shadow validation (n=50 prompts) found 0/50 unprovenanced direct material properties. Adversarial suite v0→v3 progression: 57%→71%→79%→86% (Figure 4, Supplementary S5-S7).",
       "supported": "strong"
     },
     {
       "claim": "SMACT achieves 90.8% accuracy with zero false positives on composition validity, while LLMs show precision collapse (77.8-78.8%).",
       "evidence": "TRINITY Gold benchmark (n=2,087 compositions, 3 runs per model-format pair) with confusion matrices in Figure 3. SMACT: 100% precision, 87.1% recall vs GPT-4o: 77.8% precision, 70.9% recall.",
       "supported": "strong"
     },
     {
       "claim": "LLM accuracy on composition validity is format-dependent, improving 20-25 percentage points with ICSD-style formatting.",
       "evidence": "GPT-4o improved from 64.5% to 89.4%, Gemini from 71.1% to 90.8% when switching from PyMatgen to ICSD format. Tokenisation analysis ruled out mid-element fragmentation as explanation (Supplementary S4).",
       "supported": "strong"
     },
     {
       "claim": "Adaptive mode discovered a thermodynamically stable phase (Na4Mn2Si2O8) below the convex hull in the battery cathode task.",
       "evidence": "Reported as 7 meV/atom below convex hull in Task 2 results. However, this is a single run with no replication or uncertainty quantification on the energy calculation.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "Crystalyse is a provenance-enforced single-agent system for computational materials design that orchestrates composition screening (SMACT), crystal structure generation (Chemeleon), and ML force-field evaluation (MACE-MP0) via MCP. On a TRINITY Gold benchmark of 2,087 compositions, domain tools (SMACT) achieve 100% precision vs 78% for standalone LLMs, with LLM performance showing strong format dependence (+20-25pp with ICSD notation). Iterative prompt engineering improved adversarial robustness from 57% to 86% pass rate, with provenance enforcement eliminating material-property hallucinations in shadow validation testing.",
   "red_flags": [
     {
       "flag": "No contamination analysis for benchmark",
       "detail": "The TRINITY Gold benchmark uses publicly available ICSD and Materials Project data. GPT-4o and Gemini likely saw these compositions during training, but the paper does not address this despite discussing 'training corpus bias' in a different context."
     },
     {
       "flag": "Small adversarial test set",
       "detail": "System refinement was done on only 28 prompts (2 per category). The full 70-prompt suite is deferred to supplementary for 'independent validation' but results on it are not reported in the main paper."
     },
     {
       "flag": "Materials design tasks lack independent validation",
       "detail": "The three design tasks (quaternary oxides, Na-ion cathodes, indoor PV) produce plausible-sounding results but are not validated against DFT calculations or experimental data. The paper acknowledges these are 'plausible predictions' but presents them as demonstrations."
     }
   ],
   "cited_papers": [
     {
       "title": "ChemCrow: Augmenting large-language models with chemistry tools",
       "authors": ["A. M. Bran"],
       "year": 2023,
       "arxiv_id": "2304.05376",
       "relevance": "Seminal work on LLM tool augmentation for chemistry, directly comparable agentic approach."
     },
     {
       "title": "Autonomous chemical research with large language models",
       "authors": ["D. A. Boiko", "R. MacKnight", "B. Kline", "G. Gomes"],
       "year": 2023,
       "relevance": "Demonstrates autonomous LLM-driven chemical research (Coscientist), key prior work on scientific agents."
     },
     {
       "title": "DeepSeek-R1 incentivizes reasoning in LLMs through reinforcement learning",
       "authors": ["D. Guo"],
       "year": 2025,
       "relevance": "Reasoning LLM capabilities that underpin agentic scientific systems."
     },
     {
       "title": "Constitutional Classifiers: Defending against Universal Jailbreaks across Thousands of Hours of Red Teaming",
       "authors": ["M. Sharma"],
       "year": 2025,
       "arxiv_id": "2501.18837",
       "relevance": "Safety classifier approach referenced for future adversarial robustness improvements in agent systems."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail? A Comprehensive Taxonomy and Analysis",
       "authors": ["J. Zhang"],
       "year": 2025,
       "relevance": "Analysis of multi-agent LLM failure modes, motivates single-agent design choice in Crystalyse."
     },
     {
       "title": "Reducing Tool Hallucination via Reliability Alignment",
       "authors": ["L. Xu"],
       "year": 2024,
       "arxiv_id": "2412.04141",
       "relevance": "Addresses tool hallucination problem in LLM agents, directly relevant to provenance enforcement approach."
     },
     {
       "title": "Making Language Models Better Tool Learners with Execution Feedback",
       "authors": ["S. Qiao"],
       "year": 2024,
       "relevance": "Execution-time validation for LLM tool use, related to Crystalyse's runtime provenance approach."
     },
     {
       "title": "Kosmos: An AI Scientist for Autonomous Discovery",
       "authors": ["L. Mitchener"],
       "year": 2025,
       "relevance": "Computationally intensive multi-agent scientific discovery system, contrasted with Crystalyse's single-agent approach."
     },
     {
       "title": "AtomAgents: Alloy design and discovery through physics-aware multi-modal multi-agent artificial intelligence",
       "authors": ["A. Ghafarollahi", "M. J. Buehler"],
       "year": 2024,
       "arxiv_id": "2407.10022",
       "relevance": "Multi-agent materials design system, directly comparable to Crystalyse's single-agent approach."
     }
   ]
 }