ai-research-survey

scan.json (27840B)

---

 {
   "paper": {
     "title": "Enhancing Cross-Language Code Translation via Task-Specific Embedding Alignment in Retrieval-Augmented Generation",
     "authors": [
       "Manish Bhattarai",
       "Minh Vu",
       "Javier E. Santos",
       "Ismael Boureima",
       "Daniel O'Malley"
     ],
     "year": 2025,
     "venue": "KnowledgeNLP'25 (4th International Workshop on Knowledge-Augmented Methods for Natural Language Processing)",
     "doi": "10.18653/v1/2025.knowledgenlp-1.8"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided anywhere in the paper. The aligned embedding model and pipeline code are not released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation datasets (HPC Fortran2C++ and Numerical Recipes) and the source dataset (Stack-V2) are publicly available. However, the 25K curated Fortran snippets and generated C++ translations used for alignment are not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (256 GH200 GPUs) and model names but provides no software environment specifications, dependency lists, or version details for libraries used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. A researcher would need to reverse-engineer the pipeline from the method description."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in Figure 2 (e.g., 'Avg Aligned: 0.73 ± 0.17') and box plots in Figure 3 show distributional spread including quartiles."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests (p-values, t-tests, etc.) are reported despite claims that aligned embeddings 'significantly' improve performance over unaligned. Comparisons are based solely on mean differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative improvements are reported (14% and 15%) with baseline context (e.g., 'from 0.64 to 0.73'). Table 1 shows delta values across all configurations."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the evaluation dataset sizes (315 and 298 pairs) or the 25,000 training snippets. No power analysis or discussion of whether these sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in Figure 2 across shot configurations, and Figure 3 provides box plots showing distributional spread of CodeBLEU scores."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares aligned embeddings against unaligned (generic) embeddings using the same RAG framework, and includes zero-shot (no RAG) as an additional baseline in Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include StarEncoder (2023), and the comparison setup follows Bhattarai et al. (2024). Models tested include LLaMA 3.1 (2024) and Mixtral (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison of aligned vs. unaligned embeddings effectively ablates the alignment component. Table 1 shows zero-shot (no retrieval) vs. few-shot (with retrieval) across shot counts, isolating the contribution of both RAG and alignment."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only CodeBLEU is used as the evaluation metric. Although CodeBLEU is itself a composite of four sub-components, no additional independent metrics (e.g., compilation rate, functional correctness, BLEU, CrystalBLEU) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a small-scale manual check on a subset of translations' (Section 4) but does not report this systematically — no sample size, no inter-rater agreement, no quantitative results. The appendix shows a single qualitative example."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The embedding model is trained on Stack-V2 data, while evaluation is conducted on separate datasets: HPC Fortran2C++ (315 pairs) and Numerical Recipes (298 pairs)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides breakdowns by model (4 models), dataset (2 datasets), and shot count (0-3 shots). Figure 2 scatter plots show per-sample variation."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases are discussed. While scatter plots in Figure 2 show some samples where aligned embeddings perform worse than unaligned, these cases are not analyzed or explained."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows improvement from alignment. The observation of 'diminishing marginal gains' beyond 2 shots is the closest to a negative finding, but no configurations that hurt performance or approaches that were tried and abandoned are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of CodeBLEU improvement from 0.64→0.73 (HPC) and 0.52→0.60 (Numerical Recipes), representing 14% and 15% relative improvement, are directly supported by Figure 2 and Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that embedding alignment improves translation quality is supported by controlled comparison — aligned vs. unaligned embeddings are tested with the same models, datasets, and RAG pipeline, isolating the single variable of embedding alignment."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract and paper consistently specify 'from Fortran to C++'. The conclusion explicitly notes 'Future work could extend this alignment strategy to additional programming languages,' acknowledging the scope limitation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Section 6) discusses multiple alternative explanations: CodeBLEU may not reflect functional equivalence, contrastive learning may misalign syntactically different but functionally similar code, and noisy training data may degrade alignment."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges in Section 4 that 'CodeBLEU may not capture all functional nuances' and performed manual checks. The Limitations section extensively discusses how CodeBLEU (the proxy) may not translate to functional correctness (the desired outcome)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Models are specified with family, size, and variant: 'LLaMA 3.1-8B', 'LLaMA 3.1-70B', 'Mistral123B', 'Mixtral 8x22B' (all instruct-tuned), and 'StarCoder model with 125M parameters' for embeddings."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The actual prompts used for code translation are not provided. The appendix shows example input/output but not the prompt template or system instructions given to the LLMs."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Embedding training hyperparameters are reported (temperature τ=0.1, Adam optimizer, lr=10⁻³, batch size 128). However, LLM inference parameters (temperature, top-p, max tokens) are not stated, which significantly affect generation output."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a standard RAG pipeline (embed → retrieve → generate) without retry logic, tool use, or multi-step reasoning."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes sampling criteria ('files larger than 500 bytes', 'highest combined star and fork counts'), extraction using LLaMA 3.1-70B Instruct to isolate executable Fortran code, and translation pipeline for generating C++ pairs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is a dedicated 'Limitations' section with substantial discussion spanning multiple paragraphs covering four distinct limitation areas."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: CodeBLEU not capturing functional equivalence, contrastive learning treating syntactically different but functionally similar code as negatives, granularity issues with continuous similarity scores, and noise in training data affecting alignment quality."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope is Fortran→C++ translation. The conclusion notes 'Future work could extend this alignment strategy to additional programming languages.' Limitations state CodeBLEU doesn't capture functional behavior."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While evaluation datasets are public, the aligned embedding model, the 25K generated C++ translations, pairwise CodeBLEU matrices, and per-sample evaluation scores are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes sampling 25,000 Fortran snippets from Stack-V2 (>500 bytes, highest star/fork counts), using LLaMA 3.1-70B Instruct for Fortran extraction, and generating C++ translations with LLaMA 3.1-8B."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets (Stack-V2, HPC Fortran2C++, Numerical Recipes)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline stages are described at a high level (sample → extract → translate → compute CodeBLEU → train embeddings), but intermediate counts are missing. How many of the 500K+ Stack-V2 files survived the 500-byte and quality filtering before reaching 25K is not stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states: 'This research was funded by the LANL ASC grant AI4Coding and the LANL Institutional Computing Program, supported by the U.S. DOE NNSA under Contract No. 89233218CNA000001.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Los Alamos National Laboratory, clearly listed under the title. They evaluate open-source models (LLaMA, Mistral, Mixtral), not their own products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from DOE/NNSA for general AI4Coding research. The funder does not have a commercial stake in the specific outcome of aligned vs. unaligned embedding performance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models (LLaMA 3.1, Mistral, Mixtral). These models could have seen the evaluation benchmarks during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HPC Fortran2C++ (2023) or Numerical Recipes code pairs appeared in the training data of LLaMA 3.1, Mistral, or Mixtral models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Numerical Recipes dataset is from 1988 and its code has been widely available online for decades. HPC Fortran2C++ was published in 2023. Neither benchmark's potential presence in model training data is discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-translation cost is reported. The paper describes training time (15 min/epoch on 256 GPUs) but not the cost of using the aligned RAG system at inference time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states training was 'distributed across 256 GH200 GPUs' taking 'approximately 15 minutes per epoch, with early stopping at epoch 20.' Scaling estimates for 64 and 32 GPUs are also provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds, seed sensitivity analysis, or results across multiple seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. It is unclear whether results represent single runs or averages over multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While specific hyperparameter values are given (τ=0.1, lr=10⁻³, batch=128), no search budget, search method, or number of configurations tried is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported for all configurations tested (4 models × 2 datasets × 4 shot counts × 2 alignment conditions), not just the best. Table 1 presents the full matrix."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so correction for multiple comparisons is not applicable. The absence of statistical testing is captured by the significance_tests item."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment of potential bias from evaluating their own alignment method. They implement both the aligned and unaligned pipelines without discussing whether this introduces bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The alignment requires training on 256 GH200 GPUs for 20 epochs, but performance gains are not discussed relative to this substantial compute cost. No comparison at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses CodeBLEU's construct validity: Section 4 notes it 'may not capture all functional nuances' and the Limitations section extensively discusses the gap between CodeBLEU scores and functional equivalence."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The RAG scaffold is held constant across all comparisons — only the embedding model (aligned vs. unaligned) varies. The same retrieval pipeline, same k values, and same LLMs are used for both conditions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The Numerical Recipes code has been publicly available since 1988, and HPC Fortran2C++ since 2023. LLaMA 3.1 and other models may have seen these in training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the RAG retrieval examples leak information that would not be available in a real deployment scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Stack-V2 (used for alignment training) shares code with the HPC Fortran2C++ or Numerical Recipes evaluation sets, which could inflate alignment effectiveness."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are applied (no canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines)."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "claims": [
     {
       "claim": "Task-specific embedding alignment improves average CodeBLEU from 0.64 to 0.73 (14% relative improvement) on HPC Fortran2C++ dataset.",
       "evidence": "Figure 2 (llama3.1 70b scatter plot) and Table 1 show aligned embeddings consistently outperform unaligned across all models and shot counts on HPC Fortran2C++.",
       "supported": "moderate"
     },
     {
       "claim": "On the Numerical Recipes dataset, alignment improves average CodeBLEU from 0.52 to 0.60 (15% relative improvement).",
       "evidence": "Figure 2 (Numerical Recipe panels) and Table 1 show consistent improvement across models and shot counts.",
       "supported": "moderate"
     },
     {
       "claim": "These gains are realized without any fine-tuning of the language model.",
       "evidence": "The method description (Section 3) and experimental setup (Section 4) confirm only the embedding model is trained; the LLMs (LLaMA, Mistral, Mixtral) are used as-is.",
       "supported": "strong"
     },
     {
       "claim": "Diminishing returns are observed beyond 2-shot prompting.",
       "evidence": "Table 1 shows the gap between 2-shot and 3-shot improvements is smaller than 1-shot to 2-shot across most configurations.",
       "supported": "moderate"
     },
     {
       "claim": "Aligned embeddings reduce performance variability compared to unaligned embeddings.",
       "evidence": "Figure 3 box plots show tighter interquartile ranges for aligned configurations, and Figure 2 reports lower standard deviations (e.g., 0.17 vs 0.19 for HPC with llama3.1 70b).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper demonstrates that aligning retrieval embeddings with task-specific CodeBLEU scores via contrastive learning (S-InfoNCE) improves Fortran-to-C++ code translation quality by 14-15% on two benchmarks without fine-tuning the LLM. The approach works consistently across four different models (LLaMA 3.1-8B/70B, Mistral 123B, Mixtral 8x22B) and shows diminishing returns beyond 2-shot prompting. Larger models benefit more from alignment, and aligned embeddings also reduce output variability.",
   "red_flags": [
     {
       "flag": "Single evaluation metric",
       "detail": "The entire evaluation relies solely on CodeBLEU, which the authors themselves acknowledge may not capture functional correctness. No compilation rate, execution correctness, or other independent metrics are reported."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of 'significant improvement' are made without any formal statistical tests. With evaluation sets of only 315 and 298 pairs, the observed differences could potentially be explained by chance."
     },
     {
       "flag": "Small evaluation datasets",
       "detail": "Results are based on just 315 (HPC Fortran2C++) and 298 (Numerical Recipes) code pairs. No discussion of whether these sizes are sufficient for reliable conclusions."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Numerical Recipes code has been publicly available since 1988 and widely distributed. The evaluated LLMs may have memorized these translations during training, which could confound the results."
     },
     {
       "flag": "Massive compute without cost-benefit analysis",
       "detail": "Training requires 256 GH200 GPUs for 20 epochs, but the paper does not discuss whether the 14-15% CodeBLEU improvement justifies this compute cost, nor does it report inference-time overhead."
     }
   ],
   "cited_papers": [
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper that established the retrieval-augmented generation framework used in this work."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced Codex and HumanEval; foundational work on LLM-based code generation evaluation."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo"],
       "year": 2020,
       "relevance": "Pre-trained code embedding model used as a baseline embedding approach in code translation RAG systems."
     },
     {
       "title": "Enhancing code translation in language models with few-shot learning via retrieval-augmented generation",
       "authors": ["Manish Bhattarai", "Javier E Santos"],
       "year": 2024,
       "arxiv_id": "2407.19619",
       "relevance": "Prior work by same authors establishing the RAG-based code translation framework that this paper extends with aligned embeddings."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM whose encoder (StarEncoder 125M) is used as the embedding backbone for retrieval alignment."
     },
     {
       "title": "CodeBLEU: a method for automatic evaluation of code synthesis",
       "authors": ["Shuo Ren", "Daya Guo"],
       "year": 2020,
       "arxiv_id": "2009.10297",
       "relevance": "Defines the CodeBLEU metric used both as the evaluation metric and as the alignment signal for embedding training."
     },
     {
       "title": "StarCoder 2 and the Stack V2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Provides the Stack-V2 dataset (500K+ Fortran snippets) used as the source corpus for embedding alignment training."
     },
     {
       "title": "Granite code models: A family of open foundation models for code intelligence",
       "authors": ["Mayank Mishra", "Matt Stallone"],
       "year": 2024,
       "arxiv_id": "2405.04324",
       "relevance": "Open-source code model family; cited for task-specific alignment approaches in code intelligence."
     },
     {
       "title": "Creating a dataset for high-performance computing code translation using LLMs: A bridge between OpenMP Fortran and C++",
       "authors": ["Bin Lei", "Caiwen Ding"],
       "year": 2023,
       "relevance": "Created the HPC Fortran2C++ benchmark dataset (315 pairs) used as one of the two evaluation benchmarks in this paper."
     },
     {
       "title": "LLaMA: Open and efficient foundation language models",
       "authors": ["Hugo Touvron", "Thibaut Lavril"],
       "year": 2023,
       "arxiv_id": "2302.13971",
       "relevance": "Foundation model family (LLaMA 3.1 variants) used as both the translation LLM and the synthetic data generator."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The Fortran→C++ translation use case is practically relevant for HPC code migration, but requires 256 GPUs for alignment training and targets a niche language pair."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Task-specific embedding alignment improving RAG is intuitive rather than surprising; the contribution is in the specific mechanism (S-InfoNCE with CodeBLEU) rather than a counterintuitive finding."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with established results."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Los Alamos National Laboratory is well-known in HPC/scientific computing but not a major AI research brand."
     }
   }
 }