ai-research-survey

scan.json (24519B)

---

 {
   "paper": {
     "title": "Automated Extraction of Mechanical Constitutive Models from Scientific Literature using Large Language Models: Applications in Cultural Heritage Conservation",
     "authors": ["Rui Hu", "Yue Wu", "Tianhao Su", "Yin Wang", "Shunbo Hu", "Jizhong Huang"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.16551"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. The web platform is described but no link or source code is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The Heritage Materials Constitutive Database is described but no download link, archive, or public URL is provided. The web platform screenshots are shown but no access information is given."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, library versions, requirements files, or dependency lists are provided. The paper does not mention which specific LLM models were used (only referring generically to 'resource-efficient' and 'high-capability' models)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology section describes the framework conceptually but does not provide runnable instructions or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Only point estimates are reported: Precision 80.4%, Recall 83.3%, F1 81.9%. No confidence intervals or error bars are provided for any metric."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make comparative claims against alternative systems. It reports only its own system's performance metrics without comparing to baselines, so significance tests are not applicable."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims '90% reduction in manual curation time' but provides no baseline measurement or methodology for this estimate. The claim appears in the abstract and discussion but lacks supporting data."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The corpus of 2,000+ papers is described but no justification is given for why this sample size is adequate. The ground truth evaluation is on 113 papers (all papers passing the Gatekeeper), but no justification is given for this size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single-run results are reported. No variance across runs, seeds, or repeated evaluations is mentioned. Given LLM non-determinism (acknowledged in Section 2.4.4), variance across runs would be informative but is absent."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included. The paper mentions traditional NLP pipelines and rule-based systems in the introduction as being inadequate but does not compare its system against any of them quantitatively."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so this criterion cannot be satisfied. The paper references traditional approaches (regex, NER, rule-based) only in prose without any quantitative comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (Gatekeeper agent, Analyst agent, Context-Aware Symbolic Grounding, Schema-Constrained Decoding, Self-Correction) but no ablation study is conducted to measure which components contribute to performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Precision (80.4%), Recall (83.3%), F1-Score (81.9%), and AUC (0.782), along with a full confusion matrix including TP, TN, FP, FN counts."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Domain experts in solid mechanics manually annotated the entire 113-paper set to create a ground truth dataset of 222 target constitutive model entities (Section 3.2). System outputs were evaluated against this expert ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no separation of development and test sets. The ground truth was created from the same 113 papers the system processed. No held-out set was used; all evaluation is on the same data used for system development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides a distribution of extracted constitutive models by mechanical mechanism category (Elasto-Plasticity 31.9%, Failure & Damage 24.8%, Time-dependent/Rheology 12.4%, etc.). However, per-category precision/recall breakdowns are not provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 discusses that residual errors are 'largely attributable to the parsing challenges of non-standard tabular structures in older PDFs.' Section 5 discusses limitations with older low-resolution scanned documents and challenges with chart-to-data conversion."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. The paper does not mention any approaches that were tried and failed, configurations that did not work, or design decisions that led to worse performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: 113 core documents from 2,000+ papers (Section 3.1), 185 model instances and 450+ parameters (Section 3.1), 80.4% precision (Section 3.2), and ~90% workload reduction (Section 5, though this last claim is weakly supported)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the framework 'reduces manual data curation time by approximately 90%' (abstract) without any controlled comparison or measurement methodology. This is a causal claim ('reduces') with no supporting study design. The paper also claims the Symbolic Grounding mechanism resolves ambiguities but provides only a single qualitative example."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Applications in Cultural Heritage Conservation' broadly, and the conclusion states this 'lays the foundation for the Digital Material Twin of our built heritage.' However, the corpus was sourced exclusively from arXiv (Section 2.2.1), which is not a primary venue for heritage conservation research. This significant source limitation is not acknowledged as bounding the generalizability of results."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, the 80.4% precision could be partially explained by the restrictive Gatekeeper filtering creating an easier extraction task on pre-selected relevant papers, but this is not considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "No specific LLM model names or versions are stated anywhere in the paper. The Gatekeeper is described as a 'resource-efficient Large Language Model' and the Analyst as having 'advanced reasoning capabilities,' but no model names (e.g., GPT-4, Claude, etc.) or version identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes what the prompts do in natural language (e.g., 'evaluating each document against three non-negotiable dimensions') but does not provide any actual prompt text. No appendix with prompts is included."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported — no temperature, top-p, max tokens, or any LLM API settings are mentioned. The Head-Truncation Strategy mentions '8,000 characters' but no model inference parameters are given."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The two-stage agentic scaffolding is described in detail: Gatekeeper for relevance filtering (Section 2.3), Analyst for extraction (Section 2.4), including Context-Aware Target Identification (2.4.1), Symbolic Grounding (2.4.2), Schema-Constrained Decoding (2.4.3), and Error Handling with Self-Correction (2.4.4). The pipeline workflow is documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the data acquisition process: sourcing from arXiv with boolean search queries combining heritage and mechanics keywords, PDF parsing with a custom pipeline, and a Head-Truncation Strategy extracting ~8,000 characters for Stage I processing."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section (Section 5) contains substantive discussion of limitations, including sensitivity to older low-resolution scanned documents and challenges with multimodal extraction from complex scientific charts."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations discussed in Section 5 are specific to the system's technical capabilities (document quality, chart extraction) but do not address threats to the validity of the evaluation itself — e.g., the lack of inter-annotator agreement for the ground truth, the single-source (arXiv) corpus bias, or the absence of held-out evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not acknowledge that results are limited to arXiv papers only (not the broader heritage conservation literature), nor that the evaluation is on a single corpus without cross-validation. The discussion focuses on future improvements rather than explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the raw corpus of 2,000+ papers, the 113 filtered papers, the expert annotations (ground truth), nor the extracted database are made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2.1 describes the data collection: papers sourced from arXiv, targeting physics and engineering categories, using boolean search combining heritage keywords with mechanics terms. The filtering pipeline (2,000+ → 113) is described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for a study. The domain experts who created the ground truth annotations are not described as study participants — they appear to be the research team itself. This is a benchmark evaluation paper, not a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: 2,000+ papers ingested → PDF parsing and serialization (Section 2.2.2) → Gatekeeper filtering to 113 papers (Section 2.3) → Analyst extraction yielding 185 model instances and 450+ parameters (Section 3.1). Filtering counts at each stage are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This work is supported by the Advanced Materials-National Science and Technology Major Project(2025ZD0618802) and the Shanghai Technical Service Center of Science and Engineering Computing, Shanghai University.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed with their affiliation: Shanghai University, Shanghai, China. The corresponding author email is provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is a national science and technology project and a university computing center. Neither has a commercial stake in the outcomes of this LLM extraction system."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. It uses LLMs as extraction tools on novel domain-specific documents, not as test subjects on a known benchmark. Contamination of extraction targets in training data is not the same concern."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning: this is an information extraction pipeline, not a benchmark evaluation of model knowledge. The LLMs are used as tools, not evaluated on memorized knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning: no benchmark evaluation of pre-trained model capability is conducted."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in a study. The paper is a system/benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in a study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in a study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in a study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper emphasizes computational efficiency as a design goal (the two-stage architecture is motivated by cost reduction) but never reports actual inference costs, API spend, tokens consumed, or wall-clock time for processing the 2,000+ paper corpus."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. No GPU hours, API costs, hardware specifications, or processing time are reported despite the system processing 2,000+ papers."
       }
     }
   },
   "claims": [
     {
       "claim": "The system achieved extraction precision of 80.4%, recall of 83.3%, and F1-score of 81.9% on the 113-paper ground truth corpus.",
       "evidence": "Section 3.2 reports these metrics based on manual expert annotation of 222 ground truth constitutive model entities. Confusion matrix in Figure 3a shows TP=185, FP=45, FN=37, TN=1311.",
       "supported": "moderate"
     },
     {
       "claim": "The framework successfully extracted 185 constitutive model instances and over 450 calibrated parameters from 113 core documents filtered from 2,000+ papers.",
       "evidence": "Section 3.1 reports these numbers. Figure 2 shows the distribution of extracted models by mechanism category.",
       "supported": "moderate"
     },
     {
       "claim": "The system reduces manual data curation time by approximately 90%.",
       "evidence": "Stated in the abstract and discussed in Section 5. No measurement methodology, timing data, or controlled comparison is provided to support this specific figure.",
       "supported": "weak"
     },
     {
       "claim": "The Context-Aware Symbolic Grounding mechanism correctly resolves mathematical symbol ambiguities.",
       "evidence": "Section 3.3 provides a single qualitative case study (kaolinite clay rheology) showing the system correctly resolved an ambiguous table header scaling factor. No systematic quantitative evaluation of symbolic grounding accuracy is provided.",
       "supported": "weak"
     },
     {
       "claim": "The ROC curve yields an AUC of 0.782 with a low False Positive Rate of 3.3%.",
       "evidence": "Figure 3b shows the ROC curve. However, the text mentions AUC of 0.782 while the figure caption says 0.762, creating an inconsistency.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The paper presents a two-stage LLM agentic framework for extracting mechanical constitutive models from scientific literature about cultural heritage materials. From a corpus of 2,000+ arXiv papers, the Gatekeeper agent filtered to 113 relevant documents, from which the Analyst agent extracted 185 constitutive model instances with 80.4% precision and 83.3% recall. A qualitative case study demonstrates the system's ability to resolve symbol ambiguities using domain context. The system is deployed as a web-based knowledge retrieval platform for heritage conservation researchers.",
   "red_flags": [
     {
       "flag": "No model identities disclosed",
       "detail": "The paper never names which LLMs are used for the Gatekeeper or Analyst agents. This is a fundamental reproducibility gap — the results cannot be replicated without knowing what models were used."
     },
     {
       "flag": "No baselines",
       "detail": "No comparison against any baseline system (rule-based, NER-based, or alternative LLM approaches). The paper criticizes traditional approaches in the introduction but never quantitatively compares against them."
     },
     {
       "flag": "AUC inconsistency",
       "detail": "The text in Section 3.2 states AUC of 0.782 while Figure 3b caption states AUC of 0.762. This discrepancy is unexplained."
     },
     {
       "flag": "Unsupported 90% workload reduction claim",
       "detail": "The claim that the system reduces manual curation time by ~90% appears in the abstract but is never measured or justified with any timing data or user study."
     },
     {
       "flag": "Single-source corpus bias",
       "detail": "All papers are sourced exclusively from arXiv, which is not a primary venue for heritage conservation or materials science research. This creates a significant corpus bias that is never acknowledged."
     },
     {
       "flag": "No variance across LLM runs",
       "detail": "The paper acknowledges LLM non-determinism (Section 2.4.4, self-correction mechanism) but reports only single-run results without any variance or stability analysis."
     },
     {
       "flag": "No prompts or hyperparameters disclosed",
       "detail": "Neither the actual prompts used for the Gatekeeper or Analyst agents nor any LLM API parameters (temperature, etc.) are provided. Combined with the missing model identities, the methodology is essentially a black box."
     }
   ],
   "cited_papers": [
     {
       "title": "14 examples of how LLMs can transform materials science and chemistry: a reflection on a large language model hackathon",
       "authors": ["K. M. Jablonka"],
       "year": 2023,
       "relevance": "Explores LLM applications in materials science, directly relevant to understanding LLM capabilities for scientific domain extraction."
     },
     {
       "title": "Scientific discovery in the age of artificial intelligence",
       "authors": ["H. Wang"],
       "year": 2023,
       "relevance": "Broad survey of AI capabilities for scientific discovery, relevant to understanding LLM applications in research automation."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["J. Wei"],
       "year": 2022,
       "relevance": "Foundational work on LLM reasoning capabilities that underlies the prompting approach used in this extraction framework."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["T. Brown"],
       "year": 2020,
       "relevance": "Foundational GPT-3 paper establishing few-shot learning capabilities used in LLM-based extraction tasks."
     },
     {
       "title": "Large language models are zero-shot reasoners",
       "authors": ["T. Kojima"],
       "year": 2022,
       "relevance": "Establishes zero-shot reasoning capability of LLMs, relevant to the extraction framework's use of LLMs without task-specific training."
     },
     {
       "title": "Structured information extraction from complex scientific text with fine-tuned large language models",
       "authors": ["A. Dunn"],
       "year": 2022,
       "arxiv_id": "2212.05238",
       "relevance": "Directly relevant prior work on using LLMs for structured extraction from scientific text."
     },
     {
       "title": "Survey of hallucination in natural language generation",
       "authors": ["Z. Ji"],
       "year": 2023,
       "relevance": "Addresses LLM hallucination risks that are central to the reliability concerns of automated extraction systems."
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "authors": ["Z. Xi"],
       "year": 2025,
       "relevance": "Survey of LLM-based agents relevant to understanding the agentic framework design used in this paper."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["L. Chen"],
       "year": 2023,
       "relevance": "Addresses cost-efficiency in LLM usage, directly relevant to the paper's two-stage cost-optimization strategy."
     },
     {
       "title": "Efficient guided generation for large language models",
       "authors": ["B. T. Willard", "R. Louf"],
       "year": 2023,
       "arxiv_id": "2307.09702",
       "relevance": "Foundation for the schema-constrained decoding approach used in the Analyst agent."
     },
     {
       "title": "Nougat: Neural optical understanding for academic documents",
       "authors": ["L. Blecher"],
       "year": 2023,
       "arxiv_id": "2308.13418",
       "relevance": "PDF document parsing tool relevant to the paper's document ingestion pipeline."
     }
   ]
 }