ai-research-survey

scan.json (26439B)

---

 {
   "paper": {
     "title": "ATLAS: Artifact Generation Through Layered Constraints and LLM × MDE Synergy",
     "authors": [
       "Tong Ma",
       "Hui Lai",
       "Hui Wang",
       "Zhenhu Tian",
       "Jizhou Wang",
       "Haichao Wu",
       "Yongfan Gao",
       "Chaochao Li",
       "Fengjie Xu",
       "Ling Fang"
     ],
     "year": 2025,
     "arxiv_id": "2510.25890"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or archive is provided in the paper. The paper describes a complex multi-component framework (UMM, ICM, CVG, AGR) but does not release source code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation datasets (60 AUTOSAR components for RQ1, 20 AUTOSAR systems for RQ2, 60 Brussels I bis cases for RQ3) are not released. No download links or data repositories are mentioned."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using DeepSeek-R1-Distill-Qwen-32B served through vLLM and GPT-5 via API, but provides no requirements.txt, Dockerfile, or detailed environment setup section listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology sections describe the framework architecture in detail but do not provide instructions for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 2-7 report only point estimates (e.g., XSD pass rates, latency values, legal correctness scores). No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ATLAS outperforms baselines across multiple metrics (e.g., Legal-Correct@1: 0.467 vs 0.300 vs 0.200) but provides no statistical significance tests to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results with baseline context throughout. For example, RQ1: 0% vs 100% XSD pass rate; RQ3: Legal Correctness 0.200 (Baseline) vs 0.300 (RAG) vs 0.467 (ATLAS); Citation Precision 0.250 vs 0.428 vs 0.778. These provide enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for sample sizes: 60 AUTOSAR components (RQ1), 20 systems (RQ2), or 60 legal cases (RQ3). No power analysis or explanation of why these numbers are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "RQ1 uses three fixed random seeds (42, 1001, 20250701) but does not report variance or standard deviation across seeds. RQ2 and RQ3 use single-shot generation with no repeated runs. No spread measures are reported anywhere."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 includes four pipeline configurations (vLLM, vLLM+RAG, vLLM+RAG+GBNF, vLLM+RAG+JSON Schema) plus an LLM-API baseline (GPT-5). RQ3 compares Baseline (LLM-only), RAG, and ATLAS."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary approaches: vLLM with RAG, grammar-constrained decoding (GBNF), JSON Schema constrained decoding, and GPT-5 as a frontier model baseline. These represent current state-of-the-art approaches in constrained LLM generation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline configurations in RQ1 serve as an ablation study: vLLM (no constraints), +RAG (retrieval), +GBNF (grammar constraints), +JSON Schema (schema constraints). Each adds a component incrementally. RQ3 similarly ablates LLM-only vs RAG vs full ATLAS."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used across all RQs. RQ1: XSD pass rate, evidence coverage, latency, token counts. RQ2: file completeness, XSD pass rate, reference resolution rate, SHACL/SMT pass rates, expert review scores. RQ3: Legal Correctness, Citation Precision, Abstention Quality, Promotion Accuracy, Schema Compliance, Rule Satisfaction."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 includes expert review across four dimensions (requirements alignment, architectural quality, engineering quality, toolchain integration) with structured 5-point scale scoring and qualitative findings. RQ3 legal cases were drafted and validated by a subject-matter expert."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no mention of train/test split or held-out data. The 60 AUTOSAR components, 20 systems, and 60 legal cases are all used directly for evaluation without any separation of development and test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 provides breakdowns by complexity tier (Simple/Middle/Complex) in Tables 5 and 6. RQ1 provides breakdowns by prompt regime (Min/Std/Full) in Tables 2 and 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are extensively discussed. RQ2 describes specific failure patterns: client-server role confusion in Middle-tier systems, broken SecOC authentication chains and inverted health monitoring patterns in Complex-tier systems (Section 4.4.2, Table 6). The 0% SMT pass rate is analyzed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple negative results: 0% SMT pass rate across all tiers in RQ2, low SHACL pass rates (3.5%-14.2%), architectural quality degrading to 1/5 for Complex systems. The paper frames these honestly as demonstrating the limits of current LLMs."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims ATLAS produces 'structurally valid, auditable artifacts' and 'substantially reduce manual remediation effort, validating a graduated automation paradigm.' Results support structural validity (100% XSD pass rates) and the graduated automation claim is supported by the expert review findings showing semantic failures require human intervention."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims like 'Layer-1 enforcement reliably solves the syntax problem' backed by controlled ablation: comparing the same base model with and without constraints. The ablation design (adding one component at a time to the same vLLM backend) constitutes controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful about generalization boundaries. RQ3's legal evaluation is explicitly described as 'a feasibility probe testing architectural transfer, not claiming exhaustive legal coverage or deployment readiness.' The Limitations section states 'does not claim unsupervised certification' and identifies specific scalability limitations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses distribution distortion as an alternative explanation for constrained decoding behavior (Section 3.4), considers whether stronger base models alone could solve the problem (Section 4.4.3), and discusses the gap between reference existence and semantic correctness as a hallucination mode rather than simple structural failure."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "RQ1 specifies 'DeepSeek-R1-Distill-Qwen-32B served through vLLM' which is a specific model name/size. However, RQ2 and RQ3 say only 'GPT-5' without a version, snapshot date, or API version identifier. Marketing names without snapshot dates do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described functionally (e.g., 'buildPrompt(s, ctx, S_c)' in Algorithm 2, 'BuildRepairPrompt' in Algorithm 4) but the actual prompt text is never provided. The paper describes prompt construction procedures without revealing the actual prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature and sampling parameters are specified: RQ1 uses temperature 0.7, top-p 0.9, fixed seeds (42, 1001, 20250701). RQ2 uses temperature 0.7 for Phase 1, 0.3 for Phase 2, 0.2 for interface files. Algorithm 2 uses T=0.3, Algorithm 3 uses T=0.2."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in extensive detail across Sections 3.1-3.6: the UMM-ICM-CVG pipeline, dual-channel constraint extraction, prefix-safe automaton execution, two-layer enforcement, and audit-guided repair with two routes (automated and human-in-the-loop). Multiple algorithms (1-4) and figures (1-10) document the workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 documents AUTOSAR domain preparation in detail: UMM construction from XSD/XMI (4.2.1), ICM extraction yielding 1,161 constraints with 90% entity linking precision and 1,045 anchored to UMM entities (4.2.2), and KG materialization with 1,000+ entities and 3,049 edges (4.2.3)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Conclusion and Future Work) contains an explicit 'Limitations' subsection listing four specific limitations: incremental L2 validation, system-level semantics at scale, formal-to-programmatic scaling, and the continued need for human review."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'In RQ2, architectural drift and dangling cross-file references appear in more complex systems'; 'ATLAS can surface these violations and drive AGR, but does not yet guarantee global semantic invariants for arbitrarily dense dependency graphs'; 'scaling [SHACL and SMT] to thousands of routine business rules introduces significant maintenance overhead.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: RQ3 is 'a feasibility probe testing architectural transfer, not claiming exhaustive legal coverage'; 'does not claim unsupervised certification. Certified deployment still requires domain experts and regulatory sign-off'; and 'direct structured-to-structured transformations that demand perfect consistency may not benefit from LLM-based generation.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (generated ARXML files, legal case inputs/outputs, audit trails, expert review forms) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is described: RQ1 uses '60 representative AUTOSAR components under three prompt regimes.' RQ2 curates '20 AUTOSAR systems across three complexity tiers' with detailed descriptions of each tier. RQ3 uses '60 expert-authored Brussels I bis jurisdiction cases spanning the full hierarchy of bases of jurisdiction.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited. The expert review in RQ2 is part of the evaluation methodology but the experts are not research participants. The evaluation datasets are synthetic/curated."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented across Section 4.2 (AUTOSAR domain preparation), with specific counts at each stage: 1,161 constraints extracted, 1,045 anchored to UMM entities (90% precision), 794 SHACL shapes, 1,005 SMT assertions compiled. The generation pipeline is documented in Section 4.3-4.5."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work was supported in part by the National Natural Science Foundation of China (NSFC) under Grant 32427801 (National Major Scientific Research Instrument Development Project).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: University of Science and Technology of China, Anhui University, and Hefei Institutes of Physical Science, Chinese Academy of Sciences. No authors are affiliated with commercial LLM or AUTOSAR companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is NSFC (National Natural Science Foundation of China), a government research funding agency with no financial stake in the outcomes of this framework evaluation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses DeepSeek-R1-Distill-Qwen-32B and GPT-5 but does not state the training data cutoff date for either model. This is relevant because AUTOSAR specifications and Brussels I bis regulations could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether AUTOSAR specifications or Brussels I bis legal content appeared in the training data of the LLMs used. The models may have been trained on these publicly available standards."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the evaluation tasks are somewhat custom (specific AUTOSAR components and legal cases), the underlying domain knowledge (AUTOSAR specifications, Brussels I bis regulation) is publicly available and likely in training data. This potential contamination is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Expert review is part of the evaluation methodology, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports detailed latency (seconds) and token counts (input/output) for all pipeline configurations across prompt regimes. RQ2 reports token counts and wall-clock time broken down by Phase 1 (blueprint) and Phase 2 (component assembly) across complexity tiers."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. While per-run latency and token counts are reported, there is no information on hardware used (GPU type, memory), total API spend, or aggregate compute hours for the full evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "Layer-1 constrained decoding achieves 100% XSD structural validity for single-file AUTOSAR generation, while unconstrained LLM generation achieves 0% (vLLM) or 50% (GPT-5).",
       "evidence": "Table 4 (Section 4.3.2) shows 0% for vLLM, 50% for LLM-API, and 100% for vLLM+RAG, vLLM+RAG+GBNF, and vLLM+RAG+JSON Schema.",
       "supported": "strong"
     },
     {
       "claim": "ATLAS achieves 100% XSD structural validity across all complexity tiers in multi-file system generation (20 systems, 284 files).",
       "evidence": "Table 5 (Section 4.4.2) shows 100% XSD pass rate and file completeness across Simple, Middle, and Complex tiers.",
       "supported": "strong"
     },
     {
       "claim": "Despite 100% structural validity, semantic (SHACL) and logical (SMT) validation reveals severe failure modes at system scale, with 0% SMT pass rate across all tiers.",
       "evidence": "Section 4.4.2 reports SHACL pass rates of 3.5% (Simple), 14.2% (Middle), 9.5% (Complex), and 0% SMT pass rate system-wide.",
       "supported": "strong"
     },
     {
       "claim": "ATLAS improves Legal Correctness to 0.467 vs 0.300 (RAG) and 0.200 (Baseline) on 60 Brussels I bis jurisdiction cases.",
       "evidence": "Table 7 (Section 4.5.2) reports these values along with other metrics showing consistent ATLAS improvement.",
       "supported": "moderate"
     },
     {
       "claim": "The dual-channel ICM extraction process yields 1,161 constraints from AUTOSAR specifications with 90% entity linking precision.",
       "evidence": "Section 4.2.2 states: 'yielded a total of 1,161 normative constraints. The entity linking mechanism successfully anchored 1,045 of these to verified UMM entities (90% precision).'",
       "supported": "moderate"
     },
     {
       "claim": "ATLAS validates a graduated automation paradigm: automates routine 80% structural construction while surfacing 20% semantic ambiguities for expert review.",
       "evidence": "Section 4.4.3: 100% structural validity (automated) combined with expert review findings showing semantic failures require moderate-to-extensive human repair in Middle/Complex tiers.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "ATLAS combines LLMs with model-driven engineering through layered constraint enforcement: Layer-1 prefix-safe automaton decoding achieves 100% structural validity for generated AUTOSAR artifacts (vs. 0-50% for unconstrained generation), while Layer-2 semantic/logic validation reveals that semantic correctness degrades sharply at system scale (0% SMT pass rate for multi-file systems). Cross-domain transfer to legal reasoning (Brussels I bis) improves legal correctness from 0.200 (baseline) to 0.467 with the full pipeline. The results validate a graduated automation paradigm where LLMs reliably handle structural construction but require human-in-the-loop repair for complex semantic violations.",
   "red_flags": [
     {
       "flag": "No code or data release",
       "detail": "A complex multi-component framework with multiple algorithms, yet no source code, generated artifacts, evaluation datasets, or audit trail data is released. This makes independent verification impossible."
     },
     {
       "flag": "No variance or significance testing",
       "detail": "RQ1 mentions three random seeds but does not report variance across them. RQ2 and RQ3 use single-shot generation. No statistical significance tests support the comparative claims in RQ3 (e.g., 0.467 vs 0.300 legal correctness on only 60 cases)."
     },
     {
       "flag": "Expert review details lacking",
       "detail": "The expert review in RQ2 is described as 'structured protocol' but the number of experts, their qualifications, inter-rater reliability, and the actual review instrument are not specified. It appears to be a single expert's assessment."
     },
     {
       "flag": "GPT-5 used without version specification",
       "detail": "RQ2 and RQ3 use 'GPT-5' as the model but provide no version identifier, snapshot date, or API version. Given that model behavior changes across versions, this undermines reproducibility."
     },
     {
       "flag": "Potential contamination of LLMs on AUTOSAR/legal standards",
       "detail": "Both AUTOSAR specifications and Brussels I bis regulations are publicly available and likely in the training data of DeepSeek and GPT-5. The paper does not discuss whether the models' familiarity with these standards could inflate or deflate results."
     }
   ],
   "cited_papers": [
     {
       "title": "Guiding LLMs The Right Way: Fast, Non-Invasive Constrained Generation",
       "authors": ["Luca Beurer-Kellner", "Marc Fischer", "Martin Vechev"],
       "year": 2024,
       "arxiv_id": "2403.06988",
       "relevance": "DOMINO constrained decoding algorithm directly used as baseline approach for grammar-constrained LLM generation."
     },
     {
       "title": "Generating Structured Outputs from Language Models: Benchmark and Studies",
       "authors": ["Saibo Geng", "Hudson Cooper", "Michał Moskal"],
       "year": 2025,
       "arxiv_id": "2501.10868",
       "relevance": "SchemaBench/JSONSchemaBench benchmarks that characterize failure modes in structured output generation from LLMs."
     },
     {
       "title": "Grammar-Aligned Decoding",
       "authors": ["Kanghee Park", "Jiayu Wang", "Taylor Berg-Kirkpatrick", "Nadia Polikarpova", "Loris D'Antoni"],
       "year": 2025,
       "relevance": "Grammar-aligned decoding addresses distribution distortion in constrained generation, a key challenge discussed in ATLAS."
     },
     {
       "title": "Reflexion: language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao"],
       "year": 2023,
       "relevance": "Verbal reinforcement learning paradigm that ATLAS's audit-guided repair is compared to for iterative LLM refinement."
     },
     {
       "title": "Teaching Large Language Models to Self-Debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schaerli", "Denny Zhou"],
       "year": 2024,
       "relevance": "Self-debugging approach related to ATLAS's audit-guided repair loop for LLM-generated artifacts."
     },
     {
       "title": "Hallucination-Free? Assessing the Reliability of Leading AI Legal Research Tools",
       "authors": ["Varun Magesh", "Faiz Surani", "Matthew Dahl", "Mirac Suzgun", "Christopher D. Manning", "Daniel E. Ho"],
       "year": 2025,
       "relevance": "Directly motivates ATLAS's legal domain evaluation (RQ3) and the need for constraint enforcement to prevent hallucinations in legal AI."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["Hou Xinyi", "Zhao Yanjie", "Liu Yue"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs in software engineering that provides context for ATLAS's MDE+LLM integration approach."
     },
     {
       "title": "SpecGen: Automated Generation of Formal Program Specifications via Large Language Models",
       "authors": ["Lezhi Ma", "Shangqing Liu", "Yi Li", "Xiaofei Xie", "Lei Bu"],
       "year": 2025,
       "arxiv_id": "2401.08807",
       "relevance": "Related work on LLM-based formal specification generation with verification, similar to ATLAS's constraint-guided generation."
     },
     {
       "title": "A Dual-Stage Framework for Behavior-Enhanced Automated Code Generation in Industrial-Scale Meta-Models",
       "authors": ["Tong Ma", "Shenlong Dai", "Yongfan Gao", "Fengjie Xu", "Ling Fang"],
       "year": 2025,
       "doi": "10.1109/ACCESS.2025.3614174",
       "relevance": "S2D2 predecessor framework by the same first author; ATLAS extends its dual-path fusion to dynamic constraint enforcement for LLMs."
     },
     {
       "title": "No Need to Lift a Finger Anymore? Assessing the Quality of Code Generation by ChatGPT",
       "authors": ["Zhijie Liu", "Yutian Tang", "Xiapu Luo", "Yuming Zhou", "Liang Feng Zhang"],
       "year": 2024,
       "relevance": "Evaluation of LLM code generation quality relevant to understanding the baseline capabilities ATLAS extends."
     },
     {
       "title": "Code Repair with LLMs Gives an Exploration-Exploitation Tradeoff",
       "authors": ["Hao Tang", "Keya Hu", "Jin Peng Zhou"],
       "year": 2024,
       "arxiv_id": "2405.17503",
       "relevance": "LLM-based code repair approach related to ATLAS's audit-guided repair mechanism."
     }
   ]
 }