ai-research-survey

scan.json (28271B)

---

 {
   "paper": {
     "title": "Novel Preprocessing Technique for Data Embedding in Engineering Code Generation Using Large Language Model",
     "authors": [
       "Yu-Chen Lin",
       "Akhilesh Kumar",
       "Norman Chang",
       "Wenliang Zhang",
       "Muhammad Zakir",
       "Rucha Apte",
       "Haiyang He",
       "Chao Wang",
       "Jyh-Shing Roger Jang"
     ],
     "year": 2023,
     "venue": "2024 IEEE LLM Aided Design Workshop (LAD)",
     "arxiv_id": "2311.16267",
     "doi": "10.1109/LAD62341.2024.10691715"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The paper proposes four data preprocessing techniques for RAG-based domain-specific code generation: LLM-based Data Splitting, Data Renovation with CoDRC confidence scoring, Data Augmentation via script reconstruction, and the IKEC prompt technique. In a preliminary experiment on 5 RedHawk-SC MapReduce scripts, RAG+IKEC+ChatEDA achieves 73.33% 'Percentage of Correct Lines.' The paper explicitly states that the full experiment (10 planned comparison groups on 20 scripts) has not been completed, and overall method results are not yet available.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or release of any kind is mentioned in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The scripts and API documentation are proprietary to Ansys RedHawk-SC. No data is released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Azure OpenAI Studio' and 'GPT-4, temperature = 0.0, other parameters: default' but provides no requirements.txt, dependency list, or environment specification sufficient to recreate the setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The pipeline relies on proprietary scripts and internal Ansys tools that are not available to external researchers."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The 73.33% 'Percentage of Correct Lines' is a bare point estimate with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. The IKEC comparison (86.21% vs 93.10%) and the 73.33% average are presented without any tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The 73.33% is reported without a baseline comparison. The IKEC example (86.21% vs 93.10%) provides some context but is a single example, not a systematic effect size report."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The preliminary experiment uses N=5 scripts with no justification for this sample size. The paper acknowledges 'Most significant improvements have been observed in small datasets (at least five questions)' in Section 7 but offers no power analysis or rationale."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measure is reported. Only the average across 5 samples (73.33%) is given."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The preliminary experiment (Section 4.2.1) tests only RAG+IKEC+ChatEDA. Ten comparison groups including baselines (RAG alone, ReAct, etc.) are designed in Section 4.2.2 but explicitly stated as not yet completed."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "Planned baselines include RAG and ReAct which are contemporary, but since the baseline experiment has not been executed, no actual baseline comparison exists."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The planned experiment (Section 4.2.2) is an ablation design with 10 groups systematically adding components. However, the paper states 'the full experiment has not yet been completed.' Type A tests show individual components qualitatively but are not ablation studies."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only 'Percentage of Correct Lines' is used. The planned Executability and Functionality (unit test pass rate) metrics in Section 4.2.2 have not been implemented."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "For Data Augmentation, 20 generated scripts were 'manually evaluated by in-house RedHawk-SC experts' (Section 4.1.1). For the code generation examples, errors are manually marked by experts."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The 5 test scripts are used as ground truth, but it is unclear whether they overlap with the RAG data source. No explicit separation of test data from retrieval data is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "The planned experiment includes difficulty categories (easy/medium/hard) but this has not been executed. The preliminary result is a single average across 5 scripts."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 9 shows failure cases in red (missing return value, function name errors). Section 4.1.1 notes 'optimizations in code performance are still needed' for generated scripts."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The RAG-only result (without IKEC) in Fig. 9 shows clear failures: omitted layer count code, missing return value, missing MapReduce result. The paper also acknowledges incomplete experiments and small datasets."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the preprocessing techniques 'enhance the Retrieval-Augmented Generation method in retrieving more relevant information,' but the full experiment comparing with and without preprocessing has not been completed. The 73.33% result tests RAG+IKEC+ChatEDA without the proposed preprocessing pipeline."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims IKEC 'improves' performance and preprocessing techniques 'enhance' RAG. These are causal claims based on single examples (Fig. 9) and a 5-sample preliminary test without controlled comparisons or statistical evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "While the abstract bounds to 'MapReduce applications,' the conclusion claims 'Domain-specific code generation presents a challenging yet promising arena' and 'Such a method promises rapid deployment across numerous fields,' generalizing far beyond the single tool (RedHawk-SC) tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, the improvements shown in Fig. 9 could be due to prompt length differences or specific example selection, but this is not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures 'Percentage of Correct Lines' but frames this as demonstrating 'effectiveness' of the code generation method. No discussion of whether correct lines equates to functional, correct code — the paper itself plans to add executability and unit test metrics, implicitly acknowledging the proxy gap without discussing it."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4' and 'GPT-4-Turbo' without specifying versions (e.g., 'gpt-4-0613'). The Azure OpenAI Studio deployment is mentioned but no model snapshot date or API version is given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Figures 2, 3, and 8 for all major components: Data Augmentation, Data Splitter, Data Renovation, Task Planner, Script Generator, and IKEC prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is reported as 0 (or 0.0) throughout, and the paper states 'other parameters: default' (Section 4.1.1, 4.1.2). The planned full experiment also fixes temperature at 0 with 'all other parameters set to their default values.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The RAG pipeline with Data Splitter, Data Renovation, CoDRC, ATR Algorithm, Data Augmentation, Task Planner, and Script Generator is described in detail in Section 3 with workflow diagrams (Fig. 1 and Fig. 2)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is the main contribution. Sections 3.1-3.6 detail Data Augmentation, Data Splitter, Data Renovation, CoDRC, and ATR Algorithm with prompts, formulas, and examples."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Statement') acknowledges 'certain aspects, such as evaluation, have not yet been fully realized' and 'Most significant improvements have been observed in small datasets (at least five questions).' While not titled 'Limitations,' it provides substantive disclosure of the paper's incompleteness."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 identifies specific threats: evaluation not fully realized, results only from small datasets (N=5), and findings are preliminary. These are specific to this study's state of completion."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While the domain is bounded to RedHawk-SC/MapReduce, the conclusion makes broad claims about 'rapid deployment across numerous fields' and 'algorithm design code generation' without bounding what has not been tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The scripts, API documentation, and evaluation results are all proprietary to Ansys."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes using 23 original RedHawk-SC scripts (from Jibin John per acknowledgments), API documentation, and manually generated queries for 5 test scripts. The source and nature of the data is described."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The 23 scripts and 5 test subjects were selected from internal Ansys resources, but no systematic selection process is described. How and why these specific scripts were chosen is not explained."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The preprocessing pipeline is well-documented, but the evaluation data pipeline is not. How 5 scripts were selected from the available pool, how queries were manually created, and how error lines were counted is only briefly described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 acknowledges 'Ansys Inc. for providing resources' and the first author's footnote states 'This project was completed during the internship at Ansys.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: National Taiwan University and Ansys, Inc. The connection between Ansys employees and the RedHawk-SC tool being evaluated is visible from the affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Ansys provides resources and employs most authors. The paper demonstrates LLM-based code generation for Ansys's RedHawk-SC product, which directly benefits Ansys. The funder has a stake in positive results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for GPT-4. The paper uses GPT-4 without specifying when its training data ends."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 states: 'any publicly available information would have already been learned by LLMs, and thus the datasets must be generated by the researchers themselves to avoid any potential biases.' The paper explicitly chose a proprietary domain (RedHawk-SC) to avoid training data overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper argues the test data is in a domain 'where LLMs have not been trained' (Section 1), using proprietary scripts and custom-generated queries specifically to avoid contamination. While no formal verification is performed, the conceptual risk is addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Expert reviewers evaluated outputs but were not study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or latency figures are reported despite the method making multiple GPT-4 calls per code generation (Data Splitter, Renovation, CoDRC, Task Planner, Script Generator)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The total cost of running the multi-stage GPT-4 pipeline is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple seeds or sensitivity analysis. Temperature is set to 0 (deterministic), but no analysis of whether results are sensitive to this choice or other stochastic factors."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 5 scripts were used as test subjects but does not state how many times each code generation was run. With temperature=0 this may be deterministic, but this is not explicitly addressed."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. Temperature=0 and default parameters are used without justification for why these settings were chosen or whether alternatives were explored."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Fixed configuration (temperature=0, defaults) is used without justification. The planned experiment mentions adjusting 'chunk_size' and 'similarity_top_k' but this has not been executed."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose and evaluate their own method without acknowledging self-evaluation bias. The qualitative examples in Type A tests (Figs. 5-9) are author-selected."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-compute analysis. The multi-stage pipeline (multiple GPT-4 calls) is not compared against simpler approaches at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "'Percentage of Correct Lines' is used as the sole metric without discussing whether it measures code generation quality. The paper implicitly acknowledges this gap by planning executability and unit test metrics in Section 4.2.2, but does not discuss construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The scaffold (RAG pipeline) IS the contribution being tested. Only one model (GPT-4) is used, so model-scaffold confounding is not a concern."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper argues RedHawk-SC is a novel domain for LLMs, it does not formally discuss temporal leakage — e.g., whether any RedHawk-SC documentation was available online before GPT-4's training cutoff."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the RAG retrieval process leaks answer information into the generation context. The method intentionally provides relevant documentation, but the boundary between 'retrieval' and 'leakage' in evaluation is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 5 test scripts are drawn from the same set of 23 RedHawk-SC scripts used for Data Augmentation training. No discussion of whether the generated augmented scripts introduce overlap with test data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is used. The reliance on a proprietary domain is argued conceptually but not verified empirically."
       }
     }
   },
   "claims": [
     {
       "claim": "The method achieves 73.33% 'Percentage of Correct Lines' for code generation in MapReduce applications.",
       "evidence": "Section 4.2.1: Average across 5 generated code samples using RAG+IKEC+ChatEDA workflow. Individual examples show 86.21% (RAG only) and 93.10% (RAG+IKEC).",
       "supported": "weak"
     },
     {
       "claim": "Data Augmentation generates syntactically correct scripts with appropriate structure.",
       "evidence": "Section 4.1.1: 20 new scripts generated from 23 originals, manually evaluated by RedHawk-SC experts. 'All syntax is correct, and the code is generated following the LLM's self-defined objectives.'",
       "supported": "moderate"
     },
     {
       "claim": "IKEC improves code generation quality by avoiding fatal logic errors.",
       "evidence": "Section 4.1.4 and Fig. 9: Single example showing RAG-only has 3 errors (86.21%) while RAG+IKEC has 2 errors (93.10%). Specific improvements: correct layer count, return dictionary, MapReduce result.",
       "supported": "weak"
     },
     {
       "claim": "Data Renovation provides reliable expanded content for concise technical documents.",
       "evidence": "Section 4.1.3 and Fig. 7: Single example showing 'get_current_heatmap' chunk expanded with parameter usage and return value descriptions.",
       "supported": "weak"
     },
     {
       "claim": "The preprocessing techniques enhance RAG performance for domain-specific code generation.",
       "evidence": "No completed experiment tests the full preprocessing pipeline. The planned 10-group ablation (Section 4.2.2) is not executed. Only individual component demonstrations (Type A) and a partial test (RAG+IKEC only) are available.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Incomplete experiment",
       "detail": "The paper explicitly states in Section 7: 'certain aspects, such as evaluation, have not yet been fully realized' and 'experimental results for our overall method presented in this paper are yet to be obtained.' The planned 10-group ablation study on 20 scripts has not been executed."
     },
     {
       "flag": "Tiny sample size",
       "detail": "The main quantitative result (73.33%) is based on only 5 code generation tasks. Individual component demonstrations use single examples."
     },
     {
       "flag": "No baselines in quantitative experiment",
       "detail": "The 73.33% result has no comparison point — no baseline RAG-only or ReAct result is reported. Without baselines, it is impossible to assess whether the proposed method improves over alternatives."
     },
     {
       "flag": "Company evaluating own product ecosystem",
       "detail": "Most authors are Ansys employees evaluating LLM code generation for Ansys's RedHawk-SC product. Expert evaluation of generated code was conducted by in-house Ansys staff, not independent reviewers."
     },
     {
       "flag": "Cherry-picked examples",
       "detail": "Type A individual tests (Data Splitter, Renovation, IKEC) are each demonstrated with a single hand-selected example. No aggregate metrics or failure rates for these components are reported."
     },
     {
       "flag": "Claims outrun evidence",
       "detail": "The abstract presents the 73.33% result as demonstrating 'the effectiveness of our data preprocessing method,' but this result tests RAG+IKEC+ChatEDA only — the proposed preprocessing pipeline (Data Splitter, Data Renovation) is not included in the measured experiment."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundation model used in the experiments; central to evaluating LLM code generation capabilities."
     },
     {
       "title": "ChatEDA: A large language model powered autonomous agent for EDA",
       "authors": ["Zhuolun He", "Haoyuan Wu", "Xinyun Zhang", "Xufeng Yao", "Su Zheng", "Haisheng Zheng", "Bei Yu"],
       "year": 2023,
       "relevance": "Directly builds on ChatEDA's code generation workflow and data augmentation approach for domain-specific EDA code generation."
     },
     {
       "title": "An empirical evaluation of using large language models for automated unit test generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2023,
       "relevance": "TestPilot: evaluates LLM-based code generation for Mocha framework using prompt engineering without fine-tuning."
     },
     {
       "title": "VeriGen: A large language model for Verilog code generation",
       "authors": ["Shailja Thakur", "Baleegh Ahmad", "Hammond Pearce", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri", "Siddharth Garg"],
       "year": 2023,
       "arxiv_id": "2308.00708",
       "relevance": "Domain-specific code generation benchmark using fine-tuning on CodeGen-16B for Verilog, relevant to evaluating LLM code gen in specialized domains."
     },
     {
       "title": "Self-planning code generation with large language models",
       "authors": ["Xue Jiang", "Yihong Dong", "Lecheng Wang", "Zheng Fang", "Qiwei Shang", "Ge Li", "Zhi Jin", "Wenpin Jiao"],
       "year": 2023,
       "relevance": "Progressive code generation strategy that divides tasks into subtasks, integrated into this paper's workflow."
     },
     {
       "title": "AgentCoder: Multi-agent-based code generation with iterative testing and optimisation",
       "authors": ["Dong Huang", "Qingwen Bu", "Jie M. Zhang", "Michael Luck", "Heming Cui"],
       "year": 2024,
       "relevance": "Multi-agent code generation framework with iterative testing, relevant to agentic coding workflows."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Fei Xia", "Ed Chi", "Quoc V Le", "Denny Zhou"],
       "year": 2022,
       "relevance": "Foundational prompting technique that this paper's IKEC method modifies and extends."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Planned baseline method for code generation comparison; key prompting framework for complex reasoning tasks."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus", "Fabio Petroni", "Vladimir Karpukhin", "Naman Goyal"],
       "year": 2021,
       "relevance": "Foundation for the RAG technique that is central to the paper's approach."
     },
     {
       "title": "CodeGen: An open large language model for code with multi-turn program synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroaki Hayashi", "Lifu Tu", "Huan Wang", "Yingbo Zhou", "Silvio Savarese", "Caiming Xiong"],
       "year": 2023,
       "relevance": "Open-source code generation model mentioned as potential fine-tuning target for domain-specific code generation."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Major open-source LLM used by ChatEDA for fine-tuning; relevant to code generation model ecosystem."
     }
   ]
 }