ai-research-survey

scan.json (25022B)

---

 {
   "paper": {
     "title": "LLM-based and Retrieval-Augmented Control Code Generation",
     "authors": [
       "Heiko Koziolek",
       "Sten Grüner",
       "Rhaban Hark",
       "Virendra Ashiwal",
       "Sofia Linsbauer",
       "Nafise Eskandani"
     ],
     "year": 2024,
     "venue": "LLM4Code'24 (1st International Workshop on Large Language Models for Code)",
     "doi": "10.1145/3643795.3648384"
   },
   "scan_version": 2,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository is released: 'LLM-CodeGen-RAG Github Repository' [15] at https://github.com/hkoziolek/LLM-CodeGen-RAG. The paper states: 'We also publish our prompts, raw data, and source code to enable independent replication [15].'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The OSCAT function block library is open-source and publicly available. The paper states 'We also publish our prompts, raw data, and source code to enable independent replication [15],' indicating test prompts and data are in the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Specific tool versions are mentioned throughout Section 4 (GPT-4-32k version 0613, FAISS-CPU 1.7.4, text-embedding-ada-002, OpenPLC-Editor, PDFPlumberLoader), but there is no requirements.txt, Dockerfile, or dedicated environment setup section. Python version and LangChain version are not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not contain step-by-step reproduction instructions. While a GitHub repository is referenced [15], the paper itself lacks a 'Reproducing Results' section or specific commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. The evaluation consists of 3 spot-sample tests with qualitative pass/fail assessment and no quantitative metrics with uncertainty."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The paper makes implicit comparisons (RAG-augmented generation is useful) based on 3 qualitative test cases without any statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper claims 'speed up typical programming tasks significantly' (Section 8) without any quantitative measurement of time savings or comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only 3 spot-sample tests are conducted. The paper acknowledges 'these tests are merely spot samples and not exhaustive' (Section 5) but does not justify why 3 is sufficient for any claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance is reported. Each test is run once. Temperature is set to 0 to reduce randomness, but no multi-run analysis is performed to verify reproducibility."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included. There is no comparison with non-RAG generation (plain GPT-4 without function block context), manual coding, or other code generation approaches."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so the question of whether they are contemporary does not arise."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (document chunker, embedder, vector store retriever, prompt template, LLM) but no ablation study is conducted to determine which components contribute to performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No formal quantitative metrics are defined. The evaluation checks compilation success and functional correctness qualitatively on 3 examples, but these are not systematically measured across a test suite."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The authors manually inspect generated code for correctness, identify errors (e.g., wrong variable names in Test 1, function vs function block confusion in Test 2), compile the code, and verify functional correctness through OpenPLC simulation with forced input values."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no held-out test set. The prompt template was developed through 'an iterative process' (Section 4), and the same 3 tests are the only evaluation. No separation between development and evaluation data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Only 3 individual test cases are shown. There is no breakdown by category of control logic (e.g., PID control, signal processing, boolean logic) or by complexity level."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are discussed in detail: Test 1 had incorrect variable names (OUT_MAX instead of OUT) due to ambiguous specification document. Test 2 incorrectly instantiated STAIR as a function block instead of a function. Both required manual corrections."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Two of three tests produced code requiring manual corrections. The paper honestly reports these errors: incorrect variable naming from ambiguous specifications (Test 1) and function block vs function confusion (Test 2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'we successfully generated IEC 61131-3 ST code that integrated the desired function blocks, could be compiled, and validated through simulations.' However, 2 of 3 tests required manual corrections before the code could compile. The abstract omits the manual fix step, overstating the generation quality."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 8 claims 'The method can save control engineers significant time' and 'speed up typical programming tasks significantly' without any time measurement or controlled comparison. These are causal claims about productivity improvement with no supporting evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'LLM-based and Retrieval-Augmented Control Code Generation' is broad, but the evaluation uses only GPT-4, one function block library (OSCAT), and 3 hand-crafted prompts. While Section 6 discusses external validity, the claims in the abstract and conclusion are not bounded to the tested configuration."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 discusses threats to validity but focuses on generic methodological concerns (non-determinism, simple prompts, specific tooling). No specific alternative explanations for the observed results are discussed — e.g., whether GPT-4 already knows OSCAT blocks from training data, making the RAG contribution unclear."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures compilation success and functional correctness on 3 examples, but claims 'significant time savings' (Section 8) without measuring time. Section 5 acknowledges 'comparing the code generation duration with manual implementation are considered future work,' yet Section 8 still claims time savings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 specifies 'gpt-4-32k, version 0613' — a specific model version with snapshot date. Also specifies text-embedding-ada-002 for embeddings."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows the full prompt template structure, and Section 5 provides the actual queries embedded in Tests 1-3. The reader can reconstruct the complete prompts sent to GPT-4."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states: 'GPT-4's temperature parameter was set to 0 to produce as deterministic output as possible, other parameters were left on default values.' Embedding model and vector store search type are also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The RAG pipeline is described in detail: Figure 2 shows the full workflow diagram. Section 3 describes each component (Document Loader, Document Chunker, Text Embedder, Vector Store, Retriever, LLM). Section 4 specifies the exact implementations used (LangChain, PDFPlumberLoader, FAISS, RetrievalQA chain)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes how the OSCAT PDF was loaded (PDFPlumberLoader), chunked (custom regex chunker matching subsection headings), embedded (text-embedding-ada-002), and stored (FAISS-CPU 1.7.4, 4.5 MB database). Page numbers were recovered by text comparison."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' is a dedicated section discussing internal, construct, and external validity threats across several paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses threats specific to this study: non-deterministic LLM behavior addressed by temperature=0, prompts being simple compared to practice, OSCAT being open-source and rarely used commercially, OpenPLC being less complex than commercial IDEs, only 3 tests conducted."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states: 'We established internal validity only through three initial tests, but have not yet exhaustively tested the method.' Also: 'control generation prompts are still simple though not artificial, but there is more complex control logic in practice' and 'The OSCAT library is also available as open-source and rarely used in commercial systems.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We also publish our prompts, raw data, and source code to enable independent replication [15]' referencing the GitHub repository. Generated code is also shown in Figures 3, 6, 8, and 10."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes collecting 'over 50 Control Narratives from customers' and manually formulating control code generation prompts. The OSCAT library source is described (open-source, 496-page PDF, 400+ function blocks)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The data sources (OSCAT library, customer narratives) are described but do not involve participant recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3 and 4 describe the full pipeline: OSCAT PDF → PDFPlumberLoader (page-based strings) → custom regex chunker (per function block) → text-embedding-ada-002 → FAISS vector store → similarity search → prompt augmentation → GPT-4 → code output → OpenPLC import."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. All authors are from ABB Research, implying corporate funding, but no acknowledgments section or funding statement is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed with 'ABB Research, Germany' affiliation and firstname.lastname@de.abb.com email addresses. The corporate affiliation is clearly disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "ABB is a major industrial automation company that would directly benefit from automating control code generation. The method is designed for ABB's domain (PLC/DCS control logic with proprietary function block libraries). The funder has a commercial interest in the method succeeding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included. ABB Research employees developing a method for ABB's core business domain (industrial automation) have an undisclosed financial interest in the outcome."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4-32k (version 0613) to generate code but does not state its training data cutoff. This is relevant because the OSCAT library is publicly available online and could be in GPT-4's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether GPT-4 has seen OSCAT documentation during training. The paper paradoxically motivates RAG by claiming 'LLMs trained on public Internet data are not aware of such function blocks' while testing with OSCAT, an open-source library available online."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OSCAT documentation is publicly available at oscat.de since at least 2023, predating GPT-4's training. The paper does not address whether GPT-4 could already generate OSCAT-based code without RAG, which would undermine the claimed contribution of the retrieval-augmented approach."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is purely technical (code generation, compilation, simulation)."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 reports: 'Document retrieval including the call of embeddings API was performed in a sub-second time range while the delivery of the complete LLM answer took up to 20 seconds.' Wall-clock latency is provided, though no API cost in dollars or tokens consumed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. No GPU hours, total API spend, or hardware specifications are mentioned beyond the per-query latency."
       }
     }
   },
   "claims": [
     {
       "claim": "The retrieval-augmented method can integrate pre-built function blocks into LLM-generated IEC 61131-3 ST control code.",
       "evidence": "Three tests (Section 5) show GPT-4 selecting and instantiating OSCAT function blocks (SH_1, FT_AVG, GEN_SIN, STAIR, CTRL_PID, TON) based on RAG-retrieved specifications. All three produced code that compiled and ran correctly after manual corrections in 2 of 3 cases.",
       "supported": "moderate"
     },
     {
       "claim": "The generated code could be compiled and validated through simulations.",
       "evidence": "Section 5 shows OpenPLC compilation and simulation results (Figures 7, 9, 11). Test 1 required fixing two variable names, Test 2 required changing a function block call to a function call, Test 3 worked without corrections.",
       "supported": "moderate"
     },
     {
       "claim": "The method can save control engineers significant time in implementing control logic.",
       "evidence": "No time measurement or comparison with manual coding is provided. Section 5 states 'Comparing the code generation duration with manual implementation are considered future work.'",
       "supported": "unsupported"
     },
     {
       "claim": "GPT-4 correctly identified the required complex function blocks for control code generation queries supported by retrieval-augmented generation.",
       "evidence": "In all three tests (Section 5), GPT-4 selected the appropriate OSCAT function blocks (SH_1+FT_AVG, GEN_SIN+STAIR, CTRL_PID+TON) from the vector store retrieval results.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "case-study"
   ],
   "key_findings": "The paper demonstrates basic feasibility of using retrieval-augmented generation with GPT-4 to produce IEC 61131-3 Structured Text control code that integrates function blocks from the OSCAT library. In 3 spot-sample tests, GPT-4 correctly identified appropriate function blocks from vector store retrieval, but 2 of 3 tests required manual corrections (variable name errors, function vs function block confusion) before the code could compile. The errors were attributed to ambiguities in the OSCAT specification documents rather than LLM limitations.",
   "red_flags": [
     {
       "flag": "Company evaluating its own potential product",
       "detail": "All authors are from ABB Research, a major industrial automation company. The method is designed for ABB's core business domain (control code with proprietary function block libraries). No funding or competing interests disclosure."
     },
     {
       "flag": "Extremely small sample size",
       "detail": "Only 3 spot-sample tests are conducted. The paper acknowledges they are 'merely spot samples and not exhaustive' but still makes broad feasibility and time-savings claims based on them."
     },
     {
       "flag": "Abstract overstates results",
       "detail": "The abstract states code was 'successfully generated' and 'could be compiled,' but 2 of 3 tests required manual corrections before compilation was possible. The manual correction step is omitted from the abstract."
     },
     {
       "flag": "No baseline comparison",
       "detail": "No comparison with plain GPT-4 without RAG, other LLMs, or manual coding. Without a non-RAG baseline, it is impossible to assess whether the retrieval augmentation actually helps or whether GPT-4 already knows OSCAT from training data."
     },
     {
       "flag": "Contamination risk undermines contribution",
       "detail": "The paper motivates RAG by claiming LLMs don't know proprietary function blocks, but tests with OSCAT — an open-source library available online. GPT-4 may already know OSCAT blocks from training data, making the RAG contribution unclear without a non-RAG baseline."
     }
   ],
   "cited_papers": [
     {
       "title": "ChatGPT for PLC/DCS Control Logic Generation",
       "authors": ["Heiko Koziolek", "Sten Gruener", "Virendra Ashiwal"],
       "year": 2023,
       "relevance": "Directly evaluates LLM code generation capabilities for IEC 61131-3 control logic with 100 representative prompts."
     },
     {
       "title": "Expectation vs. experience: Evaluating the usability of code generation tools powered by large language models",
       "authors": ["Priyan Vaithilingam", "Tianyi Zhang", "Elena L Glassman"],
       "year": 2022,
       "relevance": "User study evaluating LLM-based code generation tool usability, relevant to developer productivity with AI coding tools."
     },
     {
       "title": "The impact of AI on developer productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Experimental study with 95 developers showing 55% time savings with GitHub Copilot, key productivity claim in the field."
     },
     {
       "title": "In-IDE code generation from natural language: Promise and challenges",
       "authors": ["Frank F Xu", "Bogdan Vasilescu", "Graham Neubig"],
       "year": 2022,
       "relevance": "Experiments with programmers on IDE-integrated code generation showing mixed results on developer workflow and code quality."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Proposes EvalPlus framework for rigorously evaluating LLM code generation correctness."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Benchmark for evaluating LLM class-level code generation, relevant to code generation evaluation methodology."
     },
     {
       "title": "Retrieval augmented code generation and summarization",
       "authors": ["Md Rizwan Parvez", "Wasi Uddin Ahmad", "Saikat Chakraborty", "Baishakhi Ray", "Kai-Wei Chang"],
       "year": 2021,
       "arxiv_id": "2108.11601",
       "relevance": "REDCODER — retrieval-augmented code generation integrating formerly written code, foundational work for RAG in code generation."
     },
     {
       "title": "ReACC: A retrieval-augmented code completion framework",
       "authors": ["Shuai Lu", "Nan Duan", "Hojae Han", "Daya Guo"],
       "year": 2022,
       "arxiv_id": "2203.07722",
       "relevance": "Retrieval-augmented code completion using external context from existing codebases."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "arxiv_id": "2302.05020",
       "relevance": "Evaluates code language models for automated program repair, relevant to LLM code generation capabilities."
     },
     {
       "title": "Code generation using machine learning: A systematic review",
       "authors": ["Enrique Dehaerne", "Bappaditya Dey", "Sandip Halder", "Stefan De Gendt", "Wannes Meert"],
       "year": 2022,
       "relevance": "Systematic review of 37 publications on ML-based code generation, relevant as a survey of the code generation field."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper proposing retrieval-augmented generation for knowledge-intensive tasks."
     }
   ]
 }