ai-research-survey

scan.json (28802B)

---

 {
   "paper": {
     "title": "Enhancing Code Translation in Language Models with Few-Shot Learning via Retrieval-Augmented Generation",
     "authors": [
       "Manish Bhattarai",
       "Javier E. Santos",
       "Shawn Jones",
       "Ayan Biswas",
       "Boian Alexandrov",
       "Daniel O'Malley"
     ],
     "year": 2024,
     "venue": "IEEE Conference on High Performance Extreme Computing (HPEC)",
     "arxiv_id": "2407.19619",
     "doi": "10.1109/HPEC62836.2024.10938485"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL, GitHub link, or archive is provided anywhere in the paper. The RAG pipeline code is not released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: Numerical Recipes (ref [20]), HPC Fortran2CPP dataset (ref [5]), and Stack-V2 (ref [21]). These are all referenced with citations to their public sources."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup is provided. The paper mentions using HuggingFace and ChromaDB but does not specify library versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided. The methods section describes the pipeline conceptually but not with enough detail to reproduce without guessing implementation decisions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table I reports mean ± standard deviation for CodeBLEU and sub-metrics across models and datasets (e.g., 'GPT-4o 0.371 ± 0.002')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RAG 'significantly improves translation quality' and models 'consistently outperformed others' but no statistical significance tests (t-tests, p-values, etc.) are reported. Comparisons are based solely on point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table II reports delta CodeBLEU scores between zero-shot and few-shot settings with absolute baseline values, giving full context for the magnitude of improvement (e.g., Granite-34B from 0.237 zero-shot to +0.363 in 1-shot)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset sizes (298 pairs, 315 pairs, 500 sampled). No power analysis or discussion of whether these sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table I reports standard deviations alongside mean scores. However, it is ambiguous whether this variance is across multiple runs or across dataset examples."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Zero-shot performance serves as the baseline for all models, with results compared against 1-shot, 2-shot, and 3-shot RAG configurations (Table II)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4o, Llama3-70B Instruct, Codestral, and Mixtral-8x22B, which were contemporary at publication time (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically varies number of shots (0, 1, 2, 3), embedding models (Nomic-Embed, Starencoder, CodeBERT), and retrieval metrics (cosine similarity vs l2 distance), effectively ablating the RAG pipeline components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table I reports CodeBLEU and its four sub-components separately: Ngram Match, Weighted Ngram Match, Syntax Tree Match, and Dataflow Match, providing multiple evaluation perspectives."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated translations is performed. All evaluation is automated using CodeBLEU. No expert review of translation quality or functional correctness."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly describe train/test splits. The RAG retrieval corpus appears to be the same as the evaluation dataset, and no leave-one-out or hold-out protocol is clearly stated."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model, dataset (HPC Fortran2CPP vs Numerical Recipes), number of shots, embedding model, and retrieval metric in Tables I-II and Figures 5-6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses StarCoder's poor performance, shows a 'bad RAG setup' experiment in Figure 5c where using the largest distance metric as retrieval degrades performance, and discusses why CodeBERT underperforms as an embedding model."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "StarCoder shows no improvement with few-shot learning (delta = 0.000 across all shots in Table II). GPT models show diminishing returns from additional shots. CodeBERT underperforms as an embedding model."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'superiority of our approach over traditional zero-shot, particularly in translating between Fortran and C++.' Table II shows positive deltas for most models in few-shot vs zero-shot, supporting this claim."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims RAG 'improves' and 'enhances' translation quality. The experimental design varies the RAG component (zero-shot vs 1/2/3-shot) while holding other factors constant (same model, same dataset, same prompts), which is adequate for this causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Enhancing Code Translation in Language Models' broadly, but all experiments are limited to Fortran→C++ translation only. The abstract partially qualifies with 'particularly in translating between Fortran and C++' but the title and framing suggest general code translation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why RAG helps. For example, it does not consider whether the improvement comes from simply having more context (longer prompts) rather than retrieval quality, or whether random example selection would perform similarly. The bad RAG experiment (Figure 5c) partially addresses retrieval quality but not alternative explanations for the overall effect."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "CodeBLEU measures textual/structural similarity to reference translations, but the paper frames results as measuring 'translation quality' and 'accuracy' without discussing the gap between textual similarity and functional correctness. No execution-based evaluation is performed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'GPT-3.5 turbo', 'GPT-4o', 'Llama3-70B Instruct', etc. No specific version identifiers, snapshot dates, or API versions are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Figures 3 (zero-shot) and 4 (few-shot), including the system prompt and user prompt structure. The variable parts are the code snippets being translated, which naturally vary per input."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No generation hyperparameters (temperature, top-p, max tokens, etc.) are reported for any of the models used. These settings significantly affect output quality."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a single-pass RAG retrieval + generation pipeline without tool use, retry logic, or agent loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes preprocessing for each dataset: 'standardized code style, removed comments, handled whitespace and special characters, and mapped Fortran subroutines to their C++ equivalents.' Stack-V2 sampling criteria (file length between 1000-10000 bytes, highest star+fork counts) are also documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion mentions 'the current limitation in Fortran-C++ pairs challenges fine-tuning LLMs and establishing benchmarks' but this is a single sentence, not substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. No mention of potential confounds, selection bias in datasets, or limitations of the experimental design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No boundaries on generalization to other language pairs, different code domains, or production settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The source datasets are publicly available, but the generated translations, CodeBLEU scores, and intermediate results are not released for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes data collection in detail: Numerical Recipes comprises 298 Fortran-C++ pairs with documented curation; HPC Fortran2CPP comprises 315 pairs from NPB, PolyBench, and DRB; Stack-V2 sampling criteria are specified (500 examples, 1000-10000 bytes, unique repos, highest star/fork counts)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks and curated datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section III and Figure 1: dataset preparation → embedding generation → retrieval → prompt construction → translation → CodeBLEU evaluation. Preprocessing steps are described for each dataset."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments section is present in the paper. All authors are from Los Alamos National Laboratory (a DOE-funded facility) but no specific funding sources or grant numbers are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All six authors are clearly listed with their Los Alamos National Laboratory affiliations and division names."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The authors are from a government national laboratory (LANL) and are evaluating third-party models (GPT, Llama, etc.). The presumed funder (DOE/US government) has no financial stake in which LLM performs best for code translation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (GPT-4o, Llama3-70B, etc.). This is critical since the Numerical Recipes book has been public since 1988 and HPC benchmarks are widely available online."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation datasets (particularly Numerical Recipes code, published since 1988, or public HPC benchmarks) were included in the training data of the LLMs being evaluated."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Numerical Recipes dataset is derived from a book published in 1988; NPB, PolyBench, and DRB benchmarks are long-standing public code. Stack-V2 is explicitly from GitHub. All are likely in the training data of models like GPT-4o. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a purely computational benchmark evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported. The paper uses both commercial API models (GPT-3.5, GPT-4o) and large open models (70B+ parameters) without reporting any cost information."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget, GPU hours, hardware specifications, or total API spend are mentioned anywhere in the paper."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The ± values in Table I appear to represent variance across dataset examples, not across random seeds or multiple runs. No explicit seed sensitivity analysis is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. It is unclear whether results represent single runs or averages over multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Generation parameters (temperature, etc.) are not even reported, let alone a search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All configurations are reported (Tables I-II, Figures 5-6) across all models, shots, embeddings, and retrieval metrics. The paper does not cherry-pick a single best configuration."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many model/configuration comparisons are made with claims of superiority, but no statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors are evaluating their own RAG method against a zero-shot baseline. No discussion of self-evaluation bias or independent replication."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "RAG adds retrieval overhead (embedding generation, vector search, longer prompts) compared to zero-shot. Using 3-shot means 3x more context than 0-shot. These compute differences are not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "CodeBLEU measures textual and structural similarity to reference translations, which is not the same as functional correctness. The paper does not discuss whether CodeBLEU actually measures code translation quality, nor compare with execution-based metrics."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The same RAG pipeline and prompt templates are used consistently across all model comparisons. The scaffold (retrieval mechanism, prompt format) is held constant when comparing models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Numerical Recipes code has been public since 1988; HPC benchmark codes (NPB, PolyBench, DRB) are long-standing public repositories. Models trained on internet data almost certainly saw these codes."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. In the RAG setting, retrieved examples from the same corpus may provide near-identical solutions as hints."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test examples are independent. The HPC Fortran2CPP dataset shows clear cluster structure (Figure 2a), suggesting code from the same benchmark suite may be highly similar."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, decontamination, or temporal splits."
       }
     }
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "claims": [
     {
       "claim": "RAG-based few-shot learning significantly improves code translation quality over zero-shot for most LLMs.",
       "evidence": "Table II shows positive CodeBLEU deltas for 7 of 9 models on HPC Fortran2CPP (e.g., Granite-34B improves from 0.237 to 0.600 with 1-shot). Similar improvements on Numerical Recipes.",
       "supported": "moderate"
     },
     {
       "claim": "Higher RAG similarity scores between retrieved examples and the query correlate with higher CodeBLEU scores.",
       "evidence": "Figure 5a-b shows this correlation visually. Figure 5c shows performance degrades when using the largest (worst) distance metric for retrieval.",
       "supported": "moderate"
     },
     {
       "claim": "Granite-34B, Llama3-70B, and Mixtral-8x22B are the top performers in few-shot settings.",
       "evidence": "Tables I-II and Figure 6 show these models achieving highest CodeBLEU improvements from RAG across both datasets. Granite-34B shows +0.363, Mixtral +0.288 on HPC Fortran2CPP 1-shot.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 Turbo and GPT-3.5 Turbo excel in zero-shot settings but show limited incremental improvement with more shots.",
       "evidence": "Table I shows GPT models have highest zero-shot scores (0.371, 0.367). Table II shows their deltas are smaller than open models in relative terms, though still positive.",
       "supported": "moderate"
     },
     {
       "claim": "CodeBERT underperforms as an embedding model compared to Nomic-Embed and Starencoder due to its 512 token limit.",
       "evidence": "Section IV states CodeBERT 'consistently underperformed' and attributes this to its 512 token limit vs 8192 for the other two. No quantitative comparison table is provided for embedding model ablation.",
       "supported": "weak"
     },
     {
       "claim": "StarCoder shows no improvement from few-shot learning.",
       "evidence": "Table II shows delta = 0.000 for StarCoder across 1, 2, and 3 shots on both datasets. Authors attribute this to smaller model size and context length.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RAG-based few-shot learning improves Fortran-to-C++ code translation over zero-shot for most LLMs, with Granite-34B showing the largest gains (0.237→0.600 CodeBLEU). Higher retrieval similarity correlates with better translation quality. GPT models lead in zero-shot but gain less from few-shot, while open models like Mixtral-8x22B and Llama3-70B show substantial improvements. CodeBLEU-only evaluation without functional correctness testing limits the strength of conclusions.",
   "red_flags": [
     {
       "flag": "No execution-based evaluation",
       "detail": "All evaluation uses CodeBLEU (textual/structural similarity) only. No translated code is compiled or executed to verify functional correctness. High CodeBLEU does not guarantee the translation works."
     },
     {
       "flag": "Severe contamination risk",
       "detail": "Numerical Recipes code has been public since 1988. NPB, PolyBench, and DRB benchmarks are long-standing public code widely indexed on GitHub. Models like GPT-4o almost certainly saw these during pre-training, making zero-shot scores unreliable as true baselines."
     },
     {
       "flag": "Unclear train/test separation in RAG",
       "detail": "The retrieval corpus appears to be the same as the evaluation dataset. When translating example i, the system may retrieve near-identical examples from the same benchmark suite. The similarity matrix (Figure 2a) shows strong clusters confirming high intra-dataset similarity."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of 'superiority' and models 'outperforming' others are based solely on point comparisons without any statistical testing across the large number of model/configuration comparisons."
     },
     {
       "flag": "Missing generation hyperparameters",
       "detail": "Temperature, top-p, max tokens, and other critical generation parameters are not reported for any model. These settings significantly affect output quality and reproducibility."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced Codex and HumanEval benchmark; foundational work on LLM code generation evaluation."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained model for code used as an embedding model in this study's RAG pipeline."
     },
     {
       "title": "Creating a Dataset for High-Performance Computing Code Translation using LLMs: A Bridge Between OpenMP Fortran and C++",
       "authors": ["B. Lei", "C. Ding", "L. Chen"],
       "year": 2023,
       "relevance": "Created the HPC Fortran2CPP dataset used as a primary evaluation benchmark; demonstrated LLM fine-tuning for code translation."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["P. Lewis", "E. Perez", "A. Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper that this work builds upon for code translation."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["R. Li", "L. B. Allal", "Y. Zi"],
       "year": 2023,
       "relevance": "Open code LLM evaluated in this study; also provides the Starencoder embedding model."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open code LLM evaluated in this study for code translation capabilities."
     },
     {
       "title": "CodeBLEU: a Method for Automatic Evaluation of Code Synthesis",
       "authors": ["S. Ren", "D. Guo", "S. Lu"],
       "year": 2020,
       "arxiv_id": "2009.10297",
       "relevance": "Defines the CodeBLEU metric used as the sole evaluation measure in this study."
     },
     {
       "title": "Lost in Translation: A Study of Bugs Introduced by Large Language Models while Translating Code",
       "authors": ["R. Pan", "A. R. Ibrahimzada", "R. Krishna"],
       "year": 2023,
       "relevance": "Studies LLM-introduced bugs during code translation, providing taxonomy of translation errors and iterative prompting approaches."
     },
     {
       "title": "Granite Code Models: A Family of Open Foundation Models for Code Intelligence",
       "authors": ["M. Mishra", "M. Stallone", "G. Zhang"],
       "year": 2024,
       "arxiv_id": "2405.04324",
       "relevance": "Open code LLM evaluated in this study; one of the top performers in few-shot code translation."
     },
     {
       "title": "Mixtral of Experts",
       "authors": ["A. Q. Jiang", "A. Sablayrolles", "A. Roux"],
       "year": 2024,
       "arxiv_id": "2401.04088",
       "relevance": "Mixture-of-experts LLM evaluated in this study; among top performers for few-shot code translation."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["E. J. Hu", "Y. Shen", "P. Wallis"],
       "year": 2021,
       "relevance": "Parameter-efficient fine-tuning method discussed as an alternative to RAG for adapting LLMs to code translation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "RAG-augmented few-shot code translation is a technique practitioners could apply to legacy code migration, though no tool is released."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Results confirm the expected finding that providing relevant examples (few-shot) improves LLM performance over zero-shot."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk concerns are raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward empirical evaluation."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released for others to try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from Los Alamos National Laboratory (recognized in HPC). Paper evaluates well-known models (GPT-4o, Llama3) but is not from a major AI lab."
     }
   }
 }