ai-research-survey

calibration.json (16763B)

---

 {
   "paper_slug": "agents4plc-automating-closedloop-2024",
   "calibration_date": "2026-02-28",
   "sonnet_scan_date": "2026-02-28",
   "agreement_rate": 1.0,
   "total_questions": 50,
   "agreements": 50,
   "disagreements": 0,
   "disagreement_details": [],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link (https://github.com/Luoji-zju/Agents4PLC_release) provided in Section V-A, and a project site at https://hotbento.github.io/Agent4PLC/. These are working URLs in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 23-task benchmark with formal verification specifications is described but no explicit standalone download link or data repository is provided separately. The GitHub repo may contain it, but the paper does not confirm a data release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions LangGraph and MetaGPT frameworks and that CodeLlama 34B runs on 'a single NVIDIA A800 80GB PCIe GPU', but no requirements.txt, Dockerfile, conda environment, or library version details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper directs readers to GitHub and project site for 'more experiment details' but contains no step-by-step reproduction instructions, commands, or procedural guidance within the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables I, II, and III are raw counts and pass rates (e.g., '16 16 100.0%') with no confidence intervals, error bars, or uncertainty estimates."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Agents4PLC 'significantly outperforms' LLM4PLC based solely on comparing raw pass rate numbers, with no statistical significance tests performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. Raw performance differences are visible in tables but not contextualized as effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has only 23 programming tasks (16 easy, 7 medium). No justification is given for why 23 tasks are sufficient, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are single-run pass rates with no standard deviation, variance across runs, or spread measures. It is unclear whether experiments were repeated."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares Agents4PLC against LLM4PLC (prior work) and ChatDev (general-purpose multi-agent system) across multiple base LLMs, as shown in Table III."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "LLM4PLC (2024) and ChatDev (2024) are contemporary baselines. The base LLMs (GPT-4o, DeepSeek V2.5, CodeLlama 34B) are current models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C (RQ3) presents an ablation study systematically removing individual components (RAG, syntax hints, one-shot prompting, CoT in Debugging Agent) and measuring impact on pass rates, shown in Table II."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: syntax compilation success rate, verifiable rate, and pass rate. RQ2 additionally evaluates efficiency by counting generation attempts."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Evaluation is entirely automated via compiler checks and formal verification. Human evaluation is irrelevant to the claims, which concern automated correctness verification of generated PLC code."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "A single 23-task benchmark is used throughout for both developing and evaluating the framework. No explicit separation into development and held-out test sets is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by 'Easy' (16 tasks) and 'Medium' (7 tasks) benchmark levels across all tables. Tasks span categories including Logical Control, Mathematical Operations, Real-time Monitoring, and Process Control."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failures such as ChatDev 'occasionally produces code in unrelated languages, such as Python or C++', and discusses limitations of the debugging agent's effectiveness in certain cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table II) shows configurations performing worse: removing CoT 'noticeably reduces the framework's performance' for medium problems, one-shot prompting does not lead to substantial improvement, and RAG has 'unintended negative effects' on easy problems."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that Agents4PLC 'significantly outperforms previous methods' are supported by Table III showing higher pass rates vs. LLM4PLC across nearly all model/level combinations. The 'comprehensive benchmark' claim is supported by the 23-task description."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions (e.g., RAG improvements, CoT importance for complex debugging) are supported by the ablation study in RQ3 which systematically removes one component at a time, meeting the standard for controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to 'generate verifiable code applicable to real-world industrial applications' and highlights 'potential' for industrial use, based on only 23 tasks in ST language and 4 case studies. The title and framing suggest broad applicability beyond what the evidence supports."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for results. The authors acknowledge writing 'an extra automation program' for LLM4PLC but don't discuss how this could introduce bias. No threats-to-validity section or confound discussion is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-4o' and 'GPT-4o-mini' with only documentation URLs but no specific API version or snapshot date. These marketing names without snapshot dates do not constitute specified versions per schema criteria. 'DeepSeek V2.5' and 'CodeLlama 34B' are more specific but the GPT models are unversioned."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes that prompts 'include critical elements of PLC coding' and discusses their importance, but no actual prompt text is provided in the paper or appendix. Figure 3 shows an example interaction but not the full system prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for any LLM calls (temperature, top-p, max tokens, etc.). The paper does not state sampling settings for GPT-4o, DeepSeek V2.5, or CodeLlama 34B."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III provides detailed description of the multi-agent scaffold: Retrieval Agent, Planning Agent, Coding Agent, Debugging Agent, and Validation Agent. Figure 2 shows the architecture, Figure 3 shows a worked example, and the paper describes the loop threshold and feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark construction states 23 programming tasks with 'human-written-verified formal specifications and reference PLC code', but does not describe task selection criteria, how human verification was performed, or what inclusion/exclusion criteria were applied."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The conclusion mentions future work (expanding to additional PLC languages) but does not substantively discuss current limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats-to-validity section exists. The paper does not discuss threats such as the small benchmark size (23 tasks), potential contamination, or the implementation variable from automating LLM4PLC."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what results do NOT show. The conclusion briefly notes future plans to 'expand the framework to support additional PLC programming languages', implicitly acknowledging ST-only scope, but no explicit bounds are stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark data (23 programming tasks and formal specifications) is not explicitly released as a downloadable artifact. A GitHub link is provided but the paper does not confirm the benchmark data is available there."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Benchmark construction is described only at high level: 23 tasks covering 'Logical Control, Mathematical Operations, Real-time Monitoring, Process Control'. How tasks were identified, what sources were used, and inclusion criteria are not explained."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation study with no human participants. The data source is a newly constructed PLC benchmark, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from 'natural language requirements to human-written-verified formal specifications and reference PLC code' is mentioned but the human verification process, tools used, and annotation protocol are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure appears in the paper. One co-author (Wenhai Wang) is affiliated with UWin Tech, whose products are used in the case study, but no formal funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed in the header. Wenhai Wang's dual affiliation with UWin Tech and Zhejiang University is disclosed in the author list."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Wenhai Wang is affiliated with UWin Tech, whose software platform is used in the RQ4 case study. No funding source is disclosed, so independence cannot be assessed. The affiliation creates a potential conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement in the paper. Given Wenhai Wang's UWin Tech affiliation and the use of their commercial platform in the case study, the absence of a competing interests declaration is notable."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o, GPT-4o-mini, DeepSeek V2.5, and CodeLlama 34B but does not state training data cutoff dates for any of them. This is relevant because ST code examples may exist in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 23 benchmark tasks or their solutions could appear in the LLMs' training data. The benchmark is claimed to be newly created but no contamination analysis is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims this is the 'first benchmark dataset focused on the task of generating ST code from natural language specification', but does not verify that similar tasks or solutions are absent from LLM training data, nor discuss contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token usage, or cost per example reported for calls to GPT-4o and GPT-4o-mini. For an agentic system with multiple rounds of LLM calls per example, cost is important but not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only compute detail is that CodeLlama 34B runs on 'a single NVIDIA A800 80GB PCIe GPU'. Total compute budget (GPU hours, API spend, total experiment time) is not stated."
       }
     }
   },
   "notes": "Perfect agreement between Opus and Sonnet on all 50 questions. This paper has many clear-cut answers: most statistical and transparency questions are clearly NO (no CIs, no significance tests, no hyperparameters, no prompts, no limitations section), while evaluation design items are mostly clearly YES (baselines present, ablation study performed, multiple metrics used). The 23-task benchmark is very small, but both raters agree on how to evaluate each criterion. The paper's weaknesses are straightforward to identify, and its strengths (multi-agent architecture description, ablation study, contemporary baselines) are equally unambiguous."
 }