ai-research-survey

scan.json (29353B)

---

 {
   "paper": {
     "title": "Training LLMs for Generating IEC 61131-3 Structured Text with Online Feedback",
     "authors": [
       "Aaron Haag",
       "Bertram Fuchs",
       "Altay Kacan",
       "Oliver Lohse"
     ],
     "year": 2024,
     "venue": "2025 IEEE/ACM International Workshop on Large Language Models for Code (LLM4Code)",
     "arxiv_id": "2410.22159",
     "doi": "10.1109/LLM4Code66737.2025.00013"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes an iterative online DPO approach combining compiler feedback and LLM-based semantic expert feedback to fine-tune Phi-3-medium-instruct (14B) for IEC 61131-3 Structured Text code generation. Starting from a 7% baseline compilation rate, the method achieves ~70% compilation rate and ~45% semantic correctness after 9 DPO iterations, outperforming GPT-3.5 Turbo (40% compilation). Performance plateaus after approximately iteration 2, and the joint compilation + semantic correctness rate reaches ~39%.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses APPS (public) and OSCAT (public) as source datasets, but the converted ST code dataset (~2000 samples transpiled from Python via GPT-4) and the evaluation datasets are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'two NVIDIA GTX A6000 GPUs, providing a total of 96GB of memory' and that Phi-3 14B was used without quantization, but provides no requirements.txt, library versions, or detailed environment specification sufficient to recreate the setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Algorithm 1 gives a high-level pseudocode but lacks the specific details needed to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Fig. 3 shows only point estimates for compilation rate, semantic correctness, and joint rate across iterations. No error bars, confidence intervals, or uncertainty measures are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims their method outperforms GPT-3.5 and baseline models based solely on comparing point estimates (e.g., 70% vs 40% compilation rate) without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context: 'compilation success rate of around 70%, a substantial increase from the baseline's 7%' and comparison to GPT-3.5 at 40%. This provides sufficient context to assess the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation dataset is 200 intents (100 OSCAT + 100 converted APPS) with no justification for this size or discussion of whether it is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single runs with no indication of result stability across different random seeds or initializations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against the base model θ (Phi-3 without fine-tuning), the SFT model θsft, and GPT-3.5 Turbo as an external baseline, shown in Fig. 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only external baseline is GPT-3.5 Turbo. In 2024, GPT-4, Claude, CodeLlama, DeepSeek-Coder, and other code-specialized models exist and would be more competitive baselines. GPT-4 is used only as an evaluator, not as a code generation baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The method has two feedback components (compiler feedback and LLM semantic expert) but no ablation study isolates their individual contributions. There is no experiment showing DPO with only compiler feedback or only LLM feedback."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are defined and reported: compilation rate (Pc), semantic correctness rate (Ps), and joint compilation + semantic correctness rate (Pj), as described in Section IV.A."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Semantic evaluation is performed entirely by GPT-4 (gpt-4-1106-preview). No human evaluation of generated code is included. For a domain-specific code generation task targeting industrial automation, human expert evaluation of code quality would be highly relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Table I specifies evaluation on '100 OSCAT, 100 converted APPS intents' separate from training data, but there is no explicit statement that these are disjoint from the DPO instruction set (also drawn from OSCAT and APPS). The held-out separation is not clearly documented."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported only as aggregate metrics (Pc, Ps, Pj) across all 200 evaluation intents. No breakdown by OSCAT vs APPS, by task difficulty, or by problem category is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No specific failure cases or examples of generated code that fails are shown or analyzed. The discussion mentions generic challenges ('complex algorithms', 'edge cases') but provides no concrete failure examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "All experiments show monotonic improvement. The paper notes performance 'plateaus' after iteration 2 but does not report any configurations, approaches, or design choices that failed or hurt performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the framework 'outperforms state-of-the-art models' (plural), but results only compare against GPT-3.5 Turbo. Calling GPT-3.5 'state-of-the-art' in 2024 when GPT-4 and other stronger models exist is misleading. The abstract also claims the approach is 'highly suitable for industrial automation applications' with no real-world deployment evidence."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'The integration of the compiler feedback and the LLM expert...leads to better results' (causal language) but provides no ablation separating the effects of each component. Without isolating variables, the causal attribution is unjustified."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "While the title appropriately scopes to IEC 61131-3 ST, the conclusion claims the approach 'pav[es] the way for total automation of industry' — a claim far beyond what generating ST code on 200 test intents could support."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. For instance, the improvements could stem from increased exposure to code patterns during iterative generation rather than the DPO training signal, or the GPT-3.5 semantic expert could introduce systematic biases. None of these alternatives are considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures compilation rate and GPT-4-judged semantic correctness, then frames these as evidence the code is suitable for 'industrial automation applications.' The gap between GPT-4 judgment and actual functional correctness in PLC systems is not acknowledged. No unit tests, simulation, or hardware validation is performed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table I specifies 'Phi-3-medium-instruct LLM' as the base model. The semantic expert is 'gpt-3.5-turbo-1106' and the evaluator is 'gpt-4-1106-preview' — both include version/snapshot identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides the full expert prompt used for semantic evaluation, including system and user message structure with the specific formatting instructions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table I reports some experimental parameters (T=15 code responses, I=9 iterations, dataset sizes) but omits critical hyperparameters: learning rate, LoRA rank/alpha, batch size, temperature for code generation, DPO beta parameter, and other fine-tuning settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a training pipeline (SFT + iterative DPO), not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states implementations were 'transpiled from Python to ST codes using GPT-4, with filtering applied to exclude examples that were either too complex algorithmically or too specific to Python.' The filtering criteria are vague — 'too complex' and 'too specific to Python' are not operationalized."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section. Some limitations are discussed within the Results and Discussion section (IV.D) but they are framed as future work directions rather than explicit limitations of the current study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The discussion mentions LoRA's 'reduced parameter count may limit the model's ability' and 'the model relies on patterns learned from training data, which may not always cover edge cases.' These are framed as future improvement areas rather than threats to the validity of the current results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'the present work considers only PLC programming in the ST language' but the conclusion contradicts this by claiming the approach 'pav[es] the way for total automation of industry.' No explicit statement of what the results do NOT show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the converted ST training dataset, the preference datasets generated during DPO iterations, nor the evaluation data are released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The conversion process from APPS to ST via GPT-4 is described at a high level, but the filtering criteria ('too complex algorithmically or too specific to Python') are not operationalized. How the ~2000 resulting samples were validated or quality-checked is not described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard datasets (APPS, OSCAT)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the conversion 'resulted in ca. 2000 ST code samples' from APPS, then splits these across SFT (1000), DPO intents (900), and evaluation (100), plus OSCAT splits. But 'ca. 2000' is imprecise, the split methodology is not explained, and how filtering reduced the original APPS dataset to ~2000 is not documented with counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. The work is conducted at Siemens Technology Department but no explicit funding statement is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are disclosed as being from 'Siemens Technology Department, Munich, Germany' with Siemens email addresses."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are Siemens employees. Siemens manufactures PLCs and would benefit commercially from improved PLC code generation tools. The employer/implicit funder has a direct financial interest in positive outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included. Given all authors work for Siemens (a PLC manufacturer), disclosure of financial interests would be appropriate."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for Phi-3-medium-instruct, GPT-3.5-turbo-1106, or GPT-4-1106-preview. This is important since the evaluation uses APPS and OSCAT data that could be in the models' training sets."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "APPS was published in 2021 and OSCAT is publicly available. Phi-3 could have been trained on both. The paper does not discuss whether the base model has prior knowledge of these datasets before fine-tuning."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "APPS examples were published in 2021, well before Phi-3's training. The evaluation intents derived from APPS could have been seen during Phi-3's pre-training. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API costs (for GPT-3.5/GPT-4 queries during training and evaluation), or per-example generation time is reported. The method generates T=15 samples per intent over I=9 iterations, which incurs substantial cost not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'two NVIDIA GTX A6000 GPUs, providing a total of 96GB of memory' but does not report training time, total GPU hours, API costs for GPT-3.5 and GPT-4, or total computational budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds. Results appear to be from a single training run with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many experimental runs produced the reported results. It appears to be a single run through 9 DPO iterations."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The parameters in Table I (T=15, I=9, etc.) appear chosen without justification or search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Table I presents parameter choices (T=15 code responses, I=9 iterations, N instructions per round) without justification for how these values were selected or whether alternatives were tried."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system against their own implementation of baselines (SFT variant, base model configuration) without acknowledging potential self-comparison bias. The GPT-3.5 baseline may also be disadvantaged by not receiving ST-specific prompting."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The fine-tuned model requires significant compute (9 DPO iterations × 15 samples per intent × GPT-3.5 expert queries) beyond the baseline GPT-3.5 single-query inference. This compute asymmetry is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses intents from APPS (Python competitive programming) converted to ST and OSCAT (PLC library code). No discussion of whether these converted intents represent realistic industrial PLC programming tasks or whether the benchmark measures actual PLC code generation capability."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The comparison is between model outputs directly, not between different scaffolded systems."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "APPS was published in 2021 and OSCAT is publicly available. Phi-3 was trained after these dates. The paper does not discuss whether the model had prior exposure to these datasets or their solutions during pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The intents used for evaluation may overlap with training signal if the model has seen APPS problems during pre-training."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Training intents (100 OSCAT + 900 APPS) and evaluation intents (100 OSCAT + 100 APPS) are drawn from the same source datasets. Whether they share structural patterns, similar problem formulations, or near-duplicates is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied. No canary strings, membership inference, n-gram overlap analysis, or temporal splits are used."
       }
     }
   },
   "claims": [
     {
       "claim": "The proposed online DPO method achieves ~70% compilation success rate for ST code, up from 7% baseline.",
       "evidence": "Fig. 3A shows compilation rate Pc improving from ~7% (base model) to ~70% after 9 DPO iterations. Single-run point estimates without error bars.",
       "supported": "moderate"
     },
     {
       "claim": "Semantic correctness rate peaks at ~45% as measured by GPT-4 evaluation.",
       "evidence": "Fig. 3B shows semantic correctness Ps reaching ~45%. Evaluated by GPT-4-1106-preview, not human domain experts. No validation that GPT-4 judgments correlate with actual functional correctness.",
       "supported": "weak"
     },
     {
       "claim": "The method outperforms GPT-3.5 Turbo on ST code generation across all three metrics.",
       "evidence": "Fig. 3 shows the fine-tuned model exceeding GPT-3.5 Turbo on Pc (~70% vs ~40%), Ps (~45% vs ~25-30%), and Pj (~39% vs ~15-20%).",
       "supported": "moderate"
     },
     {
       "claim": "The framework outperforms state-of-the-art models and proves highly suitable for industrial automation applications.",
       "evidence": "Only compared against GPT-3.5 Turbo as an external baseline. No comparison against GPT-4, code-specialized models, or evaluation in industrial settings. No real-world deployment.",
       "supported": "weak"
     },
     {
       "claim": "Performance improvements are relatively constantly growing over iterations.",
       "evidence": "Fig. 3 shows most improvement occurring in iterations 1-2, then plateauing. The claim of 'constantly growing' contradicts the visible plateau in the figures.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "LLM-as-judge circular evaluation",
       "detail": "GPT-3.5 is used as the semantic expert during DPO training, and GPT-4 (same model family) is used for final evaluation. The paper claims 'cross-validation between different GPT versions' but both share similar biases and training data. No human expert validation of the semantic judgments."
     },
     {
       "flag": "Overclaiming beyond evidence",
       "detail": "The abstract claims 'outperforms state-of-the-art models' (plural) based on comparison against only GPT-3.5 Turbo. The conclusion states the approach 'pav[es] the way for total automation of industry' based on 200 evaluation intents with ~39% joint correctness."
     },
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "All authors are Siemens Technology Department employees. Siemens manufactures PLCs and would benefit commercially from PLC code generation tools. No conflict of interest or funding statement is included."
     },
     {
       "flag": "No statistical rigor",
       "detail": "All results are single-run point estimates with no error bars, no variance reporting, no significance tests, and no seed sensitivity analysis. It is impossible to assess whether the improvements are robust or the result of a lucky initialization."
     },
     {
       "flag": "Missing competitive baselines",
       "detail": "GPT-3.5 is the only external baseline despite GPT-4, CodeLlama, DeepSeek-Coder, and other code generation models being available in 2024. GPT-4 is used only as evaluator, not as a generation baseline, creating a suspiciously asymmetric comparison."
     },
     {
       "flag": "Construct validity concern",
       "detail": "Evaluation intents are derived from Python competitive programming problems (APPS) converted to ST. This may not represent realistic industrial PLC programming tasks, undermining claims about industrial applicability."
     }
   ],
   "cited_papers": [
     {
       "title": "Grounded copilot: How programmers interact with code-generating models",
       "authors": ["S. Barke", "M. B. James", "N. Polikarpova"],
       "year": 2022,
       "arxiv_id": "2206.15000",
       "relevance": "Empirical study of how developers interact with AI code generation tools, relevant to understanding AI programming productivity."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational paper on Codex/HumanEval establishing LLM code generation evaluation methodology."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["L. Ouyang", "J. Wu", "X. Jiang"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "InstructGPT/RLHF paper establishing the alignment methodology that this work builds upon."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["R. Rafailov", "A. Sharma", "E. Mitchell", "S. Ermon", "C. D. Manning", "C. Finn"],
       "year": 2023,
       "arxiv_id": "2305.18290",
       "relevance": "Core DPO method paper that this work extends to iterative online code generation training."
     },
     {
       "title": "LLM4PLC: Harnessing large language models for verifiable programming of PLCs in industrial control systems",
       "authors": ["M. Fakih", "R. Dharmaji", "Y. Moghaddas"],
       "year": 2024,
       "doi": "10.1145/3639477.3639743",
       "relevance": "Prior work on LLM-based PLC code generation with fine-tuning and verification, directly comparable approach."
     },
     {
       "title": "CodeRL: Mastering code generation through pretrained models and deep reinforcement learning",
       "authors": ["H. Le", "Y. Wang", "A. D. Gotmare", "S. Savarese", "S. C. H. Hoi"],
       "year": 2022,
       "arxiv_id": "2207.01780",
       "relevance": "RL-based code generation improvement using compiler feedback, a precursor approach to the online DPO method in this paper."
     },
     {
       "title": "StepCoder: Improve code generation with reinforcement learning from compiler feedback",
       "authors": ["S. Dou", "Y. Liu", "H. Jia"],
       "year": 2024,
       "arxiv_id": "2402.01391",
       "relevance": "Uses compiler feedback with RL for code generation improvement, closely related methodology."
     },
     {
       "title": "MapCoder: Multi-agent code generation for competitive problem solving",
       "authors": ["M. A. Islam", "M. E. Ali", "M. R. Parvez"],
       "year": 2024,
       "arxiv_id": "2405.11403",
       "relevance": "Multi-agent approach to code generation for competitive programming, relevant to LLM code generation evaluation."
     },
     {
       "title": "Code generation with AlphaCodium: From prompt engineering to flow engineering",
       "authors": ["T. Ridnik", "D. Kredo", "I. Friedman"],
       "year": 2024,
       "arxiv_id": "2401.08500",
       "relevance": "Iterative code generation approach using test feedback, related to compiler-feedback training paradigm."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["D. Hendrycks", "S. Basart", "S. Kadavath"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Defines the APPS benchmark used as the primary data source in this paper; important for code generation evaluation."
     },
     {
       "title": "Direct language model alignment from online AI feedback",
       "authors": ["S. Guo", "B. Zhang", "T. Liu"],
       "year": 2024,
       "arxiv_id": "2402.04792",
       "relevance": "Introduces the Online AI Feedback (OAIF) method that directly inspires the iterative online DPO approach in this paper."
     },
     {
       "title": "Large language monkeys: Scaling inference compute with repeated sampling",
       "authors": ["B. Brown", "J. Juravsky", "R. Ehrlich"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Shows that generating many code samples improves output quality, motivating the T=15 sampling strategy used in this paper."
     }
   ]
 }