ai-research-survey

scan.json (28200B)

---

 {
   "paper": {
     "title": "Agents4PLC: Automating Closed-loop PLC Code Generation and Verification in Industrial Control Systems using LLM-based Agents",
     "authors": [
       "Zihan Liu",
       "Ruinan Zeng",
       "Dongxia Wang",
       "Gengyun Peng",
       "Jingyi Wang",
       "Qiang Liu",
       "Peiyu Liu",
       "Wenhai Wang"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2410.14209"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link (https://github.com/Luoji-zju/Agents4PLC release) in Section V-A, and a project site at https://hotbento.github.io/Agent4PLC/. These are working URLs provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper constructs a benchmark of 23 programming tasks with formal verification specifications, but no explicit download link or repository for the benchmark data is provided separately from the code repository. It is unclear whether the benchmark is accessible as a standalone release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using LangGraph and MetaGPT frameworks, and notes that CodeLlama 34B runs on 'a single NVIDIA A800 80GB PCIe GPU', but no detailed environment specification (requirements.txt, Dockerfile, conda environment, or library versions) is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper points to a GitHub link and project site for 'more experiment details', but the paper itself contains no step-by-step reproduction instructions. The reader is directed externally without any commands or procedural guidance in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables I, II, and III are presented as raw counts and pass rates (e.g., '16 16 100.0%') with no confidence intervals, error bars, or uncertainty estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Agents4PLC 'significantly outperforms' LLM4PLC and other methods based solely on comparing pass rate numbers, with no statistical significance tests (e.g., t-tests, chi-squared, or permutation tests) performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While raw performance differences can be calculated from the tables, the paper reports no formal effect sizes (e.g., Cohen's d, odds ratios). Percentage improvements are implicit but not contextualized as effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains only 23 programming tasks (16 easy, 7 medium). No justification is given for why 23 tasks are sufficient to support generalizable claims, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as single-run pass rates with no standard deviation, variance across runs, or other spread measures. It is unclear whether experiments were repeated and averaged or run once."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares Agents4PLC against LLM4PLC (prior work) and ChatDev (a general-purpose multi-agent system) across multiple base LLMs (CodeLlama 34B, DeepSeek V2.5, GPT-4o, GPT-4o-mini), as shown in Table III."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LLM4PLC (2024) and ChatDev (2024) are contemporary baselines from the same year. The base LLMs used (GPT-4o, DeepSeek V2.5) are current models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C (RQ3) presents an ablation study removing individual components (RAG, syntax hints, one-shot prompting, CoT in the Debugging Agent) and measuring their impact on pass rates, as shown in Table II."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports three metrics: syntax compilation success rate, verifiable rate, and pass rate. Additionally, RQ2 evaluates efficiency by counting generation attempts."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation is entirely automated (compiler checks, formal verification). Human evaluation is clearly irrelevant to the claims, which concern automated correctness verification of generated PLC code."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses a single benchmark for both developing and evaluating the framework. There is no explicit separation into a development set and a held-out test set. The same 23-task benchmark appears to be used throughout."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by 'Easy' and 'Medium' benchmark levels across all tables, providing some category-level detail. The benchmark tasks span categories including Logical Control, Mathematical Operations, Real-time Monitoring, and Process Control."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failures, e.g., that ChatDev 'occasionally produces code in unrelated languages, such as Python or C++', and discusses cases where the debugging agent's effectiveness could not be fully captured. However, failure analysis is limited."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table II) shows configurations that perform worse, e.g., removing CoT 'noticeably reduces the framework's performance' for medium problems, and that one-shot prompting 'do not lead to a substantial improvement'. RAG is noted to have 'unintended negative effects' on easy problems."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims Agents4PLC 'significantly outperforms previous methods' and this is supported by Table III showing higher pass rates across nearly all model/level combinations versus LLM4PLC. Claims about the benchmark being 'comprehensive' are supported by the description of 23 tasks with formal specs."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions (e.g., 'RAG provides noticeable improvements for medium-level problems', 'CoT is crucial for handling more complex debugging tasks') are supported by the ablation study which systematically removes one component at a time, meeting the standard for controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to address 'critical challenges in PLC programming' and highlights the 'potential of our framework to generate verifiable code applicable to real-world industrial applications', based on a benchmark of only 23 tasks (16 easy, 7 medium) in ST language only. The case study covers 4 practical tasks, which is an extremely small sample for industrial applicability claims. The title and framing suggest broad applicability beyond what the evidence supports."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why Agents4PLC outperforms LLM4PLC. Notably, the paper acknowledges automating LLM4PLC with an 'extra automation program' which could introduce implementation bias. No threats-to-validity section or discussion of confounds is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-4o' and 'GPT-4o-mini' with only documentation URL references (e.g., https://platform.openai.com/docs/models/gpt-4o) but no specific API version or snapshot date. 'GPT-4o' without a snapshot date does not constitute a specified version per the schema criteria."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes that 'prompts for the Coding Agent play a crucial role' and that they 'include critical elements of PLC coding', but no actual prompt text is provided in the paper or appendix. Figure 3 shows an example interaction but not the full system prompt. The description is in natural language only."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for any LLM calls (e.g., temperature, top-p, max tokens). The paper does not state the sampling settings used for GPT-4o, DeepSeek V2.5, or CodeLlama 34B."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a detailed description of the multi-agent scaffold in Section III, including the workflow among Retrieval Agent, Planning Agent, Coding Agent, Debugging Agent, and Validation Agent. Figure 2 shows the architecture, Figure 3 shows a worked example, and the paper describes the loop threshold and feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark construction section states the dataset 'comprises 23 programming tasks' with 'human-written-verified formal specifications and reference PLC code', but does not describe how tasks were selected, what inclusion/exclusion criteria were applied, or how human verification was performed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion mentions future work but does not substantively discuss limitations of the current approach beyond noting the framework currently focuses on ST language."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity section exists. The paper does not discuss threats such as the small benchmark size (23 tasks), potential contamination of LLM training data with ST code, or the fact that LLM4PLC was automated by the authors which could introduce bias."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The conclusion briefly notes future plans to 'expand the framework to support additional PLC programming languages', implicitly acknowledging ST-only scope, but does not explicitly bound claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw benchmark data (23 programming tasks and formal specifications) is not explicitly released as a separate downloadable artifact. The GitHub link is mentioned but no benchmark data download is confirmed in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark construction is described only at a high level: 23 programming tasks covering 'Logical Control, Mathematical Operations, Real-time Monitoring, Process Control'. How tasks were identified, what sources were used, and what inclusion criteria were applied is not explained."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation study with no human participants. Data source is a newly constructed PLC programming task benchmark, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that the benchmark 'transitions from natural language requirements to human-written-verified formal specifications and reference PLC code', but the pipeline for creating and verifying these specifications (human verification process, tools used, annotation protocol) is not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure appears in the paper. One co-author (Wenhai Wang) is affiliated with both 'UWin Tech & Zhejiang University', and UWinTech products are used in the case study, but no formal funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed in the header. Wenhai Wang's dual affiliation with UWin Tech (a company whose product is used in RQ4 case studies) and Zhejiang University is disclosed in the author list."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Wenhai Wang is affiliated with UWin Tech, whose software platform (UWinTech Control Engineering Application Software) is used in the RQ4 practical case study, creating a potential conflict. No funding source is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement in the paper. Given Wenhai Wang's affiliation with UWin Tech and the use of their commercial platform in the case study, the absence of a competing interests declaration is a notable omission."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o, GPT-4o-mini, DeepSeek V2.5, and CodeLlama 34B but does not state the training data cutoff dates for any of these models. This is relevant because ST code examples may exist online that could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the 23 benchmark tasks or their solutions could have appeared in the training data of the evaluated LLMs. The benchmark tasks appear to be newly created, but no contamination analysis is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the benchmark is newly constructed ('first benchmark dataset focused on the task of generating ST code from natural language specification'), but does not verify that similar tasks or solutions are absent from the LLMs' training data, nor discuss this risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report API costs, token usage, or cost per example for calls to GPT-4o and GPT-4o-mini. For an agentic system with multiple rounds of LLM calls per example, cost is practically important but not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The only compute information given is that CodeLlama 34B runs on 'a single NVIDIA A800 80GB PCIe GPU'. Total compute budget (GPU hours, API spend, total time for experiments) is not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Agents4PLC significantly outperforms previous methods (LLM4PLC, ChatDev) in PLC code generation, achieving superior results across syntax compilation, verifiable rate, and pass rate metrics.",
       "evidence": "Table III shows Agents4PLC with GPT-4o achieves 100% syntax compilation, 68.8% verifiable rate, and 50% pass rate on Easy problems vs. LLM4PLC/GPT-4o at 87.5%, 0%, and 12.5% respectively. Section V-A discusses these results in detail.",
       "supported": "moderate"
     },
     {
       "claim": "Agents4PLC is more efficient than LLM4PLC, generating correct code more often in a single attempt.",
       "evidence": "Table I shows that Agents4PLC with DeepSeek V2.5, GPT-4o, and GPT-4o-mini achieves 100% first-attempt syntax compilation on both Easy and Medium problems, while LLM4PLC requires multiple attempts across models.",
       "supported": "moderate"
     },
     {
       "claim": "Syntax hints significantly enhance code generation effectiveness, especially for easy problems, while RAG primarily benefits medium-level problems.",
       "evidence": "Table II ablation study shows 'One-shot + Syntax Hint' achieves highest easy problem pass rate (68.8%) while 'One-shot + RAG + Syntax Hint' performs best on medium problems (28.6% pass rate).",
       "supported": "weak"
     },
     {
       "claim": "Agents4PLC can generate verifiable PLC code that meets real-world industrial control requirements, demonstrated through four practical case studies.",
       "evidence": "Section V-D (RQ4) describes four case studies (LED control, motor control, temperature/pressure monitoring, node delay monitoring) using the UWinTech platform, with qualitative descriptions of correct behavior.",
       "supported": "weak"
     },
     {
       "claim": "CoT reasoning in the Debugging Agent is crucial for handling complex (medium-level) debugging tasks.",
       "evidence": "Table II shows removing CoT reduces medium-level verifiable rate from 42.9% to 14.3%, as described in Section V-C.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "Agents4PLC is a multi-agent LLM framework for automated PLC code generation and verification using Structured Text, combining a Retrieval Agent, Planning Agent, Coding Agent, Debugging Agent, and Validation Agent with formal verification tools (nuXmv, PLCverif). Evaluated on a new 23-task benchmark, Agents4PLC with GPT-4o achieves 50% pass rate on Easy problems and 28.6% on Medium problems, substantially outperforming the prior LLM4PLC approach. An ablation study shows syntax hints and CoT-based debugging contribute most to performance, particularly for complex tasks. Four practical case studies on industrial hardware demonstrate qualitative functional correctness of generated code.",
   "red_flags": [
     {
       "flag": "Tiny benchmark (23 tasks)",
       "detail": "The entire evaluation rests on 23 programming tasks (16 Easy, 7 Medium). Pass rates derived from 7 Medium-level tasks (e.g., 2/7 = 28.6%) carry extreme statistical uncertainty, yet the paper makes broad claims about industrial applicability. No sample size justification is provided."
     },
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All results are reported as single-run pass rates with no confidence intervals, standard deviations, or significance tests. The claim that Agents4PLC 'significantly outperforms' baselines is not supported by any statistical test."
     },
     {
       "flag": "Potential conflict of interest: UWin Tech affiliation",
       "detail": "Co-corresponding author Wenhai Wang is affiliated with UWin Tech, whose commercial software platform is used for the RQ4 case study. No competing interests statement or funding disclosure is provided, making it impossible to assess independence of the evaluation."
     },
     {
       "flag": "Fairness of LLM4PLC comparison",
       "detail": "The authors state they 'write an extra automation program to drive the components of the LLM4PLC framework' to automate it. This introduces an implementation variable that is not controlled: the comparison may reflect differences in automation quality, not the underlying framework capabilities."
     },
     {
       "flag": "No model version specification",
       "detail": "GPT-4o and GPT-4o-mini are referenced only by marketing name without API version or snapshot date. Since model behavior changes across versions, results may not be reproducible."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "The evaluated LLMs (GPT-4o, DeepSeek V2.5) are trained on large code corpora that may include ST code examples similar to the benchmark tasks. No contamination analysis or discussion is provided."
     },
     {
       "flag": "No prompts released",
       "detail": "The system heavily relies on prompt engineering (syntax hints, CoT templates, RAG prompts) but no actual prompt text is provided in the paper or appendix, making the core methodology unverifiable and irreproducible from the paper alone."
     }
   ],
   "cited_papers": [
     {
       "title": "LLM4PLC: Harnessing Large Language Models for Verifiable Programming of PLCs in Industrial Control Systems",
       "authors": ["M. Fakih", "R. Dharmaji", "Y. Moghaddas", "G. Quiros", "O. Ogundare", "M. A. Al Faruque"],
       "year": 2024,
       "relevance": "The primary baseline for Agents4PLC, this paper introduces an LLM-based PLC code generation pipeline with external verification tools, representing the prior state of the art in this domain."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["C. Qian", "W. Liu", "H. Liu", "N. Chen", "Y. Dang", "J. Li", "C. Yang", "W. Chen", "Y. Su", "X. Cong"],
       "year": 2024,
       "arxiv_id": "2307.07924",
       "relevance": "A multi-agent software development system used as a baseline, representing general-purpose LLM-based code generation via agent collaboration."
     },
     {
       "title": "MetaGPT: Meta Programming for Multi-Agent Collaborative Framework",
       "authors": ["S. Hong", "X. Zheng", "J. Chen", "Y. Cheng", "J. Wang", "C. Zhang"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent framework used as infrastructure for Agents4PLC, relevant to the survey's focus on agentic code generation systems."
     },
     {
       "title": "MapCoder: Multi-Agent Code Generation for Competitive Problem Solving",
       "authors": ["M. A. Islam", "M. E. Ali", "M. R. Parvez"],
       "year": 2024,
       "arxiv_id": "2405.11403",
       "relevance": "A four-agent code generation system for competitive programming, cited as a related multi-agent approach, relevant to the survey's focus on LLM-based code generation."
     },
     {
       "title": "AutoSafeCoder: A Multi-Agent Framework for Securing LLM Code Generation through Static Analysis and Fuzz Testing",
       "authors": ["A. Nunez", "N. T. Islam", "S. K. Jha", "P. Najafirad"],
       "year": 2024,
       "arxiv_id": "2409.10737",
       "relevance": "A multi-agent framework adding security verification (static analysis and fuzzing) to code generation, related to the verification-augmented agent approach of this paper."
     },
     {
       "title": "AgentCoder: Multi-Agent-based Code Generation with Iterative Testing and Optimisation",
       "authors": ["D. Huang", "Q. Bu", "J. M. Zhang", "M. Luck", "H. Cui"],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "A three-agent code generation system with iterative testing and feedback, a related approach to the closed-loop verification in Agents4PLC."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework",
       "authors": ["Q. Wu", "G. Bansal", "J. Zhang", "Y. Wu", "B. Li", "E. Zhu", "L. Jiang", "X. Zhang", "S. Zhang", "J. Liu", "A. H. Awadallah", "R. W. White", "D. Burger", "C. Wang"],
       "year": 2024,
       "relevance": "A foundational multi-agent framework cited as infrastructure for building LLM-based multi-agent systems, directly relevant to the survey's focus on agentic workflows."
     },
     {
       "title": "Large Language Model-based Agents for Software Engineering: A Survey",
       "authors": ["J. Liu", "K. Wang", "Y. Chen", "X. Peng", "Z. Chen", "L. Zhang", "Y. Lou"],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "A survey of LLM-based agents for software engineering, directly relevant to the survey scope of agentic AI for programming tasks."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle", "S. Sootla", "I. Gat", "X. E. Tan", "Y. Adi", "J. Liu", "R. Sauvestre", "T. Remez"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "A code-specialized LLM used as a base model in Agents4PLC, relevant to the survey's coverage of code generation models."
     },
     {
       "title": "Keep the Conversation Going: Fixing 162 out of 337 Bugs for $0.42 each using ChatGPT",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "LLM-based automated code repair paper that inspired the CoT-based debugging agent design in Agents4PLC, relevant to the survey's focus on automated debugging with LLMs."
     },
     {
       "title": "A Survey on Large Language Model based Autonomous Agents",
       "authors": ["L. Wang", "C. Ma", "X. Feng", "Z. Zhang", "H. Yang", "J. Zhang"],
       "year": 2024,
       "relevance": "A comprehensive survey on LLM-based autonomous agents, directly relevant to the broader survey scope on agentic AI systems."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu", "Z. Yang", "K. Wang", "L. Li", "X. Luo", "D. Lo", "J. Grundy", "H. Wang"],
       "year": 2023,
       "relevance": "A systematic literature review on LLMs for software engineering, directly relevant to the survey scope and potentially relevant for citation chasing on methodology quality."
     }
   ]
 }