ai-research-survey

scan.json (21726B)

---

 {
   "paper": {
     "title": "Feedback Loops and Code Perturbations in LLM-based Software Engineering: A Case Study on a C-to-Rust Translation System",
     "authors": ["Martin Weiss", "Jesko Hecking-Harbusch", "Jochen Quante", "Matthias Woehrle"],
     "year": 2025,
     "venue": "arXiv preprint (submitted to IEEE)",
     "arxiv_id": "2512.02567"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The internal automotive code cannot be released, and no code for the translation system itself is shared."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark includes 30 internal automotive files that cannot be released. The 20 external files reference prior work but no dataset download link is provided. The paper states 'we are unable to release the full dataset' (Sec. IV-A)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper mentions tools (rustc, clippy, clang libFuzzer) but not their versions or configuration."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., pass@5 = 0.79). No confidence intervals or error bars are reported on the main results tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'GPT-4o mini is ~50% better than Phi-4') but no statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage point improvements with baseline context, e.g., 'first feedback loop has a high impact and can boost performance by up to 24%' and 'up to 9% points difference' between models, with baseline values visible in tables and figures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses 50 C files with 76 functions. No justification is given for why 50 files is sufficient, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Standard deviation is reported for benchmark file characteristics (Table I) but not for the experimental results themselves. Pass@k values are point estimates without spread measures across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The no-feedback-loop condition (iteration count <= 1) serves as baseline. Multiple models are compared against each other. The Identity (no perturbation) condition serves as baseline for RQ3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses GPT-4o mini, Phi-4, and reasoning models o3-mini and GPT-5 mini, which are contemporary models. GPT-3.5 Turbo is included explicitly as a historical reference."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically varies three factors (feedback loops/iterations, model choice, perturbations) and measures their individual effects. The iteration count comparison (Table III) shows the contribution of each additional feedback loop."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses pass@k for various k values, compilation success rate, and fuzzing success rate as distinct metrics. It also introduces robust pass@k (pessimistic/optimistic views) for perturbation analysis."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The system's outputs are validated by automated compilation and differential fuzzing checks. Human evaluation of translation quality is not relevant to the claims about success rates."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning training study. The benchmark is used directly for evaluation; there is no training/test split to hold out."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by internal vs. external files (Fig. 3), by perturbation level (Fig. 8), and by model (Fig. 5). Failure analysis is broken down by failure type (Fig. 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-B provides detailed error analysis including fuzzing setup failures, fuzzing exceptions, translation system errors, and LLM API errors (Table IV). Specific failure causes are discussed (type conversion issues, macro translation problems)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that large/complex files remain challenging, that LLMCodeExtraction perturbation decreases performance, and that certain internal files consistently fail. The diminishing returns of additional feedback iterations is also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims ('first feedback loop can boost performance by up to 24%', 'up to 50% with both', 'up to 9% points difference' between models) are all supported by results in Sec. V."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about feedback loops improving performance. The experimental design uses controlled single-variable manipulation (varying iteration count while holding other factors constant), which is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-C explicitly states: 'the concrete results on the translation task may not generalize to other LLM-based tasks. They may not even generalize to translation tasks between other language pairs.' The title specifies it as a case study on C-to-Rust."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses that performance differences between internal and external files 'might be due to the differences in length' and that LLMCodeExtraction perturbation's impact may be because it 'adds more complexity to the control flow.' The threats section discusses confounds from LLM randomness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-3.5 Turbo', 'GPT-4o mini', 'Phi-4', 'o3-mini', and 'GPT-5 mini' without specifying snapshot dates or API versions. These are marketing names without version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The exact prompts are provided in Sec. III: 'Translate the following C code to Rust. Keep all identifiers exactly as they are. <C code>' and the feedback prompt 'You made the following mistakes: <error messages>'. These are the actual prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B states 'All models in this paper are used with their default parameters, and we set the temperature to 0.7.' The number of runs (n=20) and max iterations (i=5) are also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The translation system's scaffolding is described in detail in Sec. III with a diagram (Fig. 1): compilation check, linting via clippy, differential fuzzing, feedback loop with error messages, and the harness generation process."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes the benchmark construction: 30 internal automotive files, 10 open-source files, 10 competitive programming files, sampled from larger sets. Table I provides file characteristics. The perturbation procedures are documented in Sec. IV-C."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-C 'Threats to Validity' provides a dedicated subsection discussing internal and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses specific issues: impact of errors on internal validity (referring to Sec. VI-B), running on only 50 C files, and that results may not generalize to other language pairs or LLM tasks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states results 'may not generalize to other LLM-based tasks' and 'may not even generalize to translation tasks between other language pairs.' It also notes the focus on 'files that capture our target domain (embedded source code).'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental data (individual run outcomes, token counts per run) is not available. The internal benchmark files cannot be released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes the three sources (30 internal automotive, 10 open-source from prior work, 10 competitive programming), the sampling approach, and Table I characterizes the files."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses code benchmarks only."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from C input through translation, compilation check, linting, fuzzing, and feedback is documented in Sec. III. The procedure for running experiments is described in Sec. IV-D. Error cases and their handling are documented in Table IV."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Three of four authors are from Bosch Research."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: one from Otto von Guericke University, three from Bosch Research."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three authors are from Bosch Research, which has a potential interest in demonstrating the viability of LLM-based code translation for automotive software. No funding disclosure is provided to assess independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the models used (GPT-3.5 Turbo, GPT-4o mini, Phi-4, o3-mini, GPT-5 mini)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that 'Keeping code internal has the benefit that LLMs should not have seen it during training' (Sec. IV-A), acknowledging the contamination concern for the 30 internal files. However, the 20 external files from prior published work are not similarly addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 20 external benchmark files are from published prior work and could have been in the training data of the models used. This is only partially addressed by noting internal code is unseen; the external code contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 and Figure 7 show token consumption versus performance trade-offs. The paper discusses cost-performance trade-offs explicitly, reporting token sums for different iteration counts and models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend, wall-clock time for the full experiment suite, or total computational budget is stated. Only relative token counts are shown in figures."
       }
     }
   },
   "claims": [
     {
       "claim": "The first feedback loop iteration boosts translation success rate by up to 24%.",
       "evidence": "Table III shows pass@5 for final result increasing from 0.69 (<=1 iteration) to 0.78 (<=2 iterations) for GPT-4o mini. Sec. V-A discusses this.",
       "supported": "strong"
     },
     {
       "claim": "Combining feedback loops and multiple runs can boost performance by up to 50%.",
       "evidence": "Sec. V-A and Fig. 2 show the combined effect of iterations and k runs on pass@k improvement.",
       "supported": "moderate"
     },
     {
       "claim": "LLM selection has up to 9 percentage points difference in performance.",
       "evidence": "Fig. 5 shows pass@k differences across GPT-3.5 Turbo, GPT-4o mini, and Phi-4. Sec. V-B discusses model differences.",
       "supported": "strong"
     },
     {
       "claim": "With feedback loops, Phi-4 becomes the best model for k in [4, 11].",
       "evidence": "Fig. 5 shows Phi-4 with 5 iterations outperforming GPT models in the k=4 to k=11 range.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o mini is robust against most code perturbations.",
       "evidence": "Fig. 8 shows most perturbations cluster around the Identity baseline. Sec. V-C discusses this finding.",
       "supported": "strong"
     },
     {
       "claim": "Feedback loops significantly improve success rates for both reasoning and non-reasoning models.",
       "evidence": "Fig. 7 shows consistent improvement patterns across GPT-4o mini, o3-mini, and GPT-5 mini with the first feedback providing ~20% improvement.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "Feedback loops in LLM-based C-to-Rust translation systems significantly improve success rates, with the first iteration providing the largest gain (up to 24%). Model selection matters but its impact is reduced when feedback loops are used; notably, the open-source Phi-4 model outperforms GPT models in the mid-range of k runs when feedback is enabled. Code perturbations have minimal impact on translation success for most perturbation types, and diversity from perturbations can be leveraged as data augmentation to boost performance by up to 9%.",
   "red_flags": [
     {
       "flag": "Small benchmark size",
       "detail": "Only 50 C files with 76 functions, with 30 being internal (unreproducible). No justification for sample size adequacy."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims between models and conditions are made based on point estimates without any statistical tests to confirm differences are not due to chance."
     },
     {
       "flag": "Unreproducible dataset",
       "detail": "60% of the benchmark (30 internal automotive files) cannot be released, making independent verification impossible."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "All models are referenced by marketing names without snapshot dates or API versions, making exact reproduction impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs for software engineering tasks, directly relevant to survey scope."
     },
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["A. Fan", "B. Gokkaya", "M. Harman"],
       "year": 2023,
       "relevance": "Major survey on LLMs in software engineering from ICSE-FoSE."
     },
     {
       "title": "Evaluating agent-based program repair at Google",
       "authors": ["P. Rondon", "R. Wei", "J. Cambronero"],
       "year": 2025,
       "relevance": "Industrial evaluation of agentic program repair, relevant to agentic AI in SE."
     },
     {
       "title": "Automated unit test improvement using large language models at Meta",
       "authors": ["N. Alshahwan", "J. Chheda", "A. Finogenova"],
       "year": 2024,
       "relevance": "Industrial-scale LLM-based test generation study."
     },
     {
       "title": "Towards translating real-world code with LLMs: A study of translating to Rust",
       "authors": ["H. F. Eniser", "H. Zhang", "C. David"],
       "year": 2024,
       "arxiv_id": "2405.11514",
       "relevance": "Directly related C-to-Rust LLM translation work with differential fuzzing approach."
     },
     {
       "title": "VERT: verified equivalent Rust transpilation with few-shot learning",
       "authors": ["A. Z. H. Yang", "Y. Takashima", "B. Paulsen"],
       "year": 2024,
       "arxiv_id": "2404.18852",
       "relevance": "C-to-Rust translation using LLMs with formal verification, related generate-and-check approach."
     },
     {
       "title": "How much does AI impact development speed? An enterprise-based randomized controlled trial",
       "authors": ["E. Paradis", "K. Grey", "Q. Madison"],
       "year": 2025,
       "relevance": "RCT measuring AI coding assistant productivity impact in enterprise setting."
     },
     {
       "title": "Do users write more insecure code with AI assistants?",
       "authors": ["N. Perry", "M. Srivastava", "D. Kumar", "D. Boneh"],
       "year": 2023,
       "relevance": "Study on security implications of AI-assisted coding."
     },
     {
       "title": "ReCode: Robustness evaluation of code generation models",
       "authors": ["S. Wang", "Z. Li", "H. Qian"],
       "year": 2023,
       "relevance": "Robustness evaluation methodology for code generation, directly used in this paper."
     },
     {
       "title": "On the robustness of code generation techniques: An empirical study on GitHub Copilot",
       "authors": ["A. Mastropaolo", "L. Pascarella", "E. Guglielmi"],
       "year": 2023,
       "relevance": "Robustness evaluation of AI code generation tools."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLM software engineering capabilities."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced pass@k metric and HumanEval benchmark, foundational to code generation evaluation."
     }
   ]
 }