ai-research-survey

scan.json (29951B)

---

 {
   "paper": {
     "title": "A Multi-Language Perspective on the Robustness of LLM Code Generation",
     "authors": ["Fazle Rabbi", "Zishuo Ding", "Jinqiu Yang"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2504.19108",
     "doi": "10.48550/arXiv.2504.19108"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM code generation robustness varies significantly across programming languages, with Java being most resilient and C++ most fragile under semantics-preserving perturbations. Semantic perturbations (docstrings, function names) cause larger performance drops than formatting changes across all models. Larger model size does not consistently improve robustness. Automated prompt repair via LLM-based docstring fixing yields only marginal recovery (~6-12% of failed cases), effective for syntactic noise but not semantic perturbations.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is released at https://github.com/frabbisw/robustextended, explicitly stated in the abstract ('We have compiled and released a dedicated dataset') and in Section 1 ('The source code and benchmark dataset are publicly available')."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The perturbed datasets for Java, C++, and JavaScript are released as part of the replication package. Section 1 contribution (4) states they 'release robustness evaluation datasets for Java, C++, and JavaScript.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using HuggingFace's AutoModelForCausalLM API (Section 4.3) but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While code is released, the paper itself does not contain a 'Reproducing Results' section or detailed commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 8-15 are reported as point estimates (e.g., RP5@1, RD5@1, RR5@1 values) without any confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 16 reports p-values from Fisher's Exact Test across models, perturbation types, and languages to test statistical significance of robustness drops."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The Robust Drop metric (RD5@1) directly quantifies the magnitude of performance decrease. Results are reported with baseline context throughout Tables 8-15 (e.g., nominal RP5@1 of 0.63 dropping to 0.42 under DocString perturbation for Magicoder-7B on Java)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for using 164 problems from HumanEval-X, why 5 perturbed versions per strategy, or any power analysis. The choice of s=5 is adopted from ReCode without independent justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. The paper uses 5 perturbed prompts per strategy but these are different inputs, not repeated runs, and no variance across these is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Nominal (unperturbed) performance serves as the baseline for measuring robustness drops. Six models are compared against each other and against prior ReCode results for Python."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The model selection includes recent models (Magicoder-7B from 2023, QwenCode-2.5-6.7B from 2024) alongside older ones (Incoder, CodeGen from 2022), providing a reasonable temporal range for comparison."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an evaluation study, not a system with components to ablate. The per-perturbation-type breakdown serves a similar purpose but is not an ablation of a proposed system."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three robustness metrics are used: Robust Pass (RPs@1), Robust Drop (RDs@1), and Robust Relative (RRs@1). Results are also reported under both worst-case (5/5) and partial-case (3/5) scenarios."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4.3 describes human annotation of 300 prompt pairs by two independent annotators, scoring naturalness and semantic similarity on a 5-point Likert scale with Cohen's kappa inter-annotator agreement reported (Table 7)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "No training or fine-tuning was performed (Section 4.3: 'We did not perform any training or fine-tuning; we directly evaluated them in their pretrained form'). The HumanEval-X and EvalPlus-X test cases are inherently held out from the evaluation pipeline."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 12-15 provide detailed per-perturbation breakdowns for all 29 perturbation types across all models and languages. Results are also broken down by perturbation category (DocString, Function Name, Syntax, Format) and by language (Java, C++, JavaScript)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figures 1, 2, and 8 show concrete failure examples. RQ2 (Section 5.2) provides feature-level analysis of what drives failures. RQ3 discusses when prompt repair fails to recover performance."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 reports that automated prompt repair 'yields modest gains limited to trivial noise and often fails on semantic perturbations, making it an unreliable robustness strategy.' They also report cases where larger models are less robust than smaller ones."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to conduct a comparative analysis of robustness across languages, introduce perturbations in four areas, and investigate LLM-based prompt repair. All three are addressed with supporting results in Sections 5.1-5.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The primary causal claims (perturbations cause performance drops) are supported by a controlled experimental design where semantics-preserving perturbations are the only variable changed. The perturbation framework ensures single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds evaluation to Java, C++, and JavaScript, excludes Python and Go with stated reasons (Section 3.2), and names the specific models tested. The title ('A Multi-Language Perspective') appropriately scopes the work rather than claiming universal LLM behavior."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 (Threats to Validity) discusses internal and external threats but only in generic terms (dataset bias, limited number of models). No specific alternative explanations for the observed results are discussed, such as whether language-specific differences in training data volume could explain the Java vs C++ robustness gap."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@k on test suites and frames results as robustness to perturbations, which directly matches the measurement. No proxy gap exists between what is measured (test pass rates under perturbation) and what is claimed (robustness degradation)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with parameter counts are provided: Incoder-1B, Incoder-6B, CodeGen-2B-Multi, CodeGen-6B-Multi, Magicoder-S-DS-6.7B, and Qwen2.5-Coder-7B-Instruct (Section 4.2). These are open-source models with fixed weights, making the names sufficient identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 9 provides the exact prompt used for docstring repair. Code generation prompts are the publicly available HumanEval-X prompts. Perturbation examples are shown in Tables 1-2 and Figures 1, 3, 5."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 reports temperature=0.2, top_p=0.95, and maximum generation length=1536 tokens. Language-specific stopping rules are also described."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated directly via HuggingFace API for code generation from prompts."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes dataset preparation in detail: converting EvalPlus test cases to Java/C++/JavaScript, updating canonical solutions, handling type incompatibilities (e.g., mixed lists excluded for Java/C++), and maintaining method signatures. Section 3.3 documents the perturbation pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides a dedicated discussion of internal and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 discusses generic threats: 'generating the same codes from the same prompt each time by the code generation model can introduce biases' and 'we use three code generation models and the human-eval-x dataset, which could introduce bias for a limited number of models.' These are boilerplate threats, not specific alternative explanations for the observed language-dependent robustness differences."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly excludes Python (covered by prior work) and Go (less commonly used) from evaluation (Section 3.2). Section 4.2 explains that certain large models were excluded due to computational constraints. The scope is bounded to specific models, languages, and perturbation types."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at https://github.com/frabbisw/robustextended includes the perturbed datasets and code, enabling independent verification of results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the data collection process: starting from HumanEval-X, converting EvalPlus test cases to multiple languages, updating canonical solutions. Section 4.1 provides dataset statistics (Table 3)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.4.3 mentions 'two independent annotators' for the human evaluation of perturbation quality but provides no information about who they are, their qualifications, or how they were selected."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from HumanEval-X through EvalPlus integration, test case conversion, canonical solution updates, and perturbation generation is documented step-by-step in Sections 3.2-3.4, with statistics on the resulting dataset in Table 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper text. University affiliations are listed but funding is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Fazle Rabbi and Jinqiu Yang at Concordia University, Zishuo Ding at The Hong Kong University of Science and Technology. No conflict with evaluated products since all models are third-party."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence since funding is not disclosed. The authors are at universities not affiliated with the evaluated models, suggesting likely independence, but without disclosure this cannot be confirmed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the six evaluated models. This is critical since HumanEval was published in 2021 and several models were trained after this date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval-X problems appeared in the training data of the evaluated models, despite HumanEval being widely known and potentially included in training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in July 2021 and is widely available online. Models like QwenCode-2.5 (2024) were trained well after this date and likely encountered HumanEval solutions during training. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation study, not a human subjects study. The two annotators in Section 4.4.3 validate perturbation quality but are not participants in a human study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study. The annotation task uses two researchers, not recruited participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study design."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study design."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study design."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study design."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study design."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, GPU hours, or wall-clock time per experiment are reported despite running six models across thousands of perturbed prompts in three languages."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions excluding large models due to computational constraints ('Due to the excessive size and our inability to run them on our machine') but never states what hardware was used or the total compute budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. The paper uses temperature=0.2 (not deterministic) but does not assess how results vary across random seeds. The 5 perturbed prompts test input variation, not seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states k=1 (single generation per prompt) and s=5 (five perturbed prompts per strategy), but does not state whether the full experiment was repeated across multiple independent runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 mentions 'chosen through a small grid search on a held-out subset' but does not report the number of configurations tried or the search budget. The values are said to be 'taken from the ReCode Wang et al. (2022),' creating a contradiction that is not resolved."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims hyperparameters were selected via grid search on a held-out subset but provides no details on the validation set, the configurations tried, or the selection criterion."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Table 16 reports 48 Fisher's exact test p-values (4 perturbation types × 4 models × 3 languages) without any correction for multiple comparisons (e.g., Bonferroni, Holm)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors constructed the EvalPlus-X dataset and perturbation pipeline, then evaluated models on their own benchmark. No discussion of potential bias from evaluating on their own dataset construction."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 1B to 7B parameters. The paper compares robustness across these scales without discussing or controlling for compute budget differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes HumanEval test cases are inadequate (motivating EvalPlus-X) but does not discuss whether function-level code completion from docstrings measures real-world code generation robustness, or whether the perturbation types reflect realistic user errors."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated directly for raw code generation from prompts."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021. Models like QwenCode-2.5 (2024) and Magicoder (2023) were trained well after the benchmark was available online. No discussion of whether models may have seen HumanEval solutions during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information that would not be available in real usage (e.g., docstring examples that partially reveal the solution)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval-X problems are independent from training data or whether similar problems exist in training corpora."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No contamination detection method is applied (no canary strings, membership inference, or n-gram overlap analysis)."
       }
     }
   },
   "claims": [
     {
       "claim": "Code generation models exhibit varying robustness across different programming languages, with Java being most resilient, followed by JavaScript, and C++ being most fragile.",
       "evidence": "Tables 8-11 show consistent patterns: Java has lowest RD5@1 values across most perturbation types and models, while C++ shows the largest drops (e.g., RD5@1 of 0.80 for C++ vs 0.31 for Java under Syntax perturbation for Incoder-1B). Section 5.1.",
       "supported": "strong"
     },
     {
       "claim": "Larger model size does not always guarantee better robustness or pass rate across all programming languages.",
       "evidence": "Tables 8 and 10 show examples where larger models are less robust (e.g., CodeGen-6B-Multi has RD5@1 of 0.64 on Java Function perturbation vs 0.40 for CodeGen-2B-Multi). Section 5.1.",
       "supported": "moderate"
     },
     {
       "claim": "Stronger test cases from EvalPlus expose substantially more failures than HumanEval-X test cases in both nominal and perturbed settings.",
       "evidence": "Comparing Tables 8 (EvalPlus) vs 9 (HumanEval-X) shows lower pass rates with EvalPlus across all models and languages. Section 5.1.",
       "supported": "strong"
     },
     {
       "claim": "Automated prompt repair via LLM-based docstring fixing yields only partial recovery, averaging ~6-12% of failed cases depending on language.",
       "evidence": "Section 5.3 and Figure 10: JavaScript recovers ~11.7% (peak 16.5%), Java ~7.4% (peak 12.2%), C++ ~6.4% (peak 11.6%). New failures are rare (4-6% of cases).",
       "supported": "moderate"
     },
     {
       "claim": "DocString and function name perturbations cause more severe robustness drops than syntax and format perturbations.",
       "evidence": "Tables 8-9 show higher RD5@1 values for DocString (up to 0.90) and Function Name perturbations compared to Format (up to 0.53) and Syntax perturbations. Figure 6 visualizes these differences.",
       "supported": "strong"
     },
     {
       "claim": "Each programming language has distinct patterns of feature-level sensitivity under perturbations.",
       "evidence": "Figure 7 heatmaps show language-specific impact scores: Java is feature-sensitive under function name and docstring perturbations but not syntax; C++ shows moderate steady correlations; JavaScript shows consistently low correlations. Section 5.2.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "HumanEval was published in 2021 and is widely available online. Models trained after 2021 (especially QwenCode-2.5 from 2024) very likely encountered HumanEval solutions during training. This could inflate nominal pass rates and affect robustness measurements, but the paper never discusses this contamination risk."
     },
     {
       "flag": "No error bars or variance across runs",
       "detail": "All results are single-run point estimates. With temperature=0.2 (not deterministic), results could vary across runs, but no variance or confidence intervals are reported for any of the thousands of measurements."
     },
     {
       "flag": "Multiple comparisons without correction",
       "detail": "Table 16 reports 48 Fisher's exact test p-values without any correction for multiple comparisons. Several p-values near the 0.05 threshold (e.g., Java DocString for Incoder-6B at 0.01) may not survive correction."
     },
     {
       "flag": "Limited and partially outdated model selection",
       "detail": "Four of six models (Incoder-1B/6B, CodeGen-2B/6B) are from 2022 and small by current standards. No state-of-the-art models (GPT-4, Claude, Llama-3) are evaluated, limiting the contemporary relevance of findings. The exclusion of larger models due to hardware constraints is acknowledged but not mitigated."
     },
     {
       "flag": "RQ3 mitigation tested with single model only",
       "detail": "The prompt repair experiment (RQ3) uses only Magicoder-7B for both repair and code generation, making it impossible to determine whether the limited recovery is due to the repair strategy or the model. Testing with multiple models would strengthen the finding."
     }
   ],
   "cited_papers": [
     {
       "title": "ReCode: Robustness Evaluation of Code Generation Models",
       "authors": ["Shiqi Wang", "Zheng Li", "Haifeng Qian"],
       "year": 2023,
       "arxiv_id": "2212.10264",
       "relevance": "Primary baseline work: first robustness evaluation benchmark for code generation models, focused on Python only."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex model, foundational for code generation evaluation."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Introduces EvalPlus, showing HumanEval test cases are inadequate and stronger tests reveal more failures in LLM-generated code."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroaki Hayashi"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "Introduces CodeGen family of code generation models evaluated in this study."
     },
     {
       "title": "CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Evaluations on HumanEval-X",
       "authors": ["Qinkai Zheng", "Xiao Xia", "Xu Zou"],
       "year": 2023,
       "arxiv_id": "2303.17568",
       "relevance": "Introduces HumanEval-X multilingual benchmark used as the base dataset for this study."
     },
     {
       "title": "Magicoder: Empowering Code Generation with OSS-Instruct",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu"],
       "year": 2023,
       "arxiv_id": "2312.02120",
       "relevance": "Introduces Magicoder model used both for code generation evaluation and as the prompt repair model in RQ3."
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "authors": ["Binyuan Hui", "Jian Yang", "Zeyu Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Technical report for QwenCode-2.5 model evaluated in this study as one of the stronger contemporary code generation models."
     },
     {
       "title": "On Robustness of Code Generation Techniques: An Empirical Study on GitHub Copilot",
       "authors": ["Antonio Mastropaolo", "Luca Pascarella", "Emanuele Guglielmi"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00181",
       "relevance": "Empirical study on robustness of GitHub Copilot's code generation, finding sensitivity to small prompt perturbations."
     },
     {
       "title": "On Robustness of Prompt-Based Semantic Parsing with Large Pre-Trained Language Model: An Empirical Study on Codex",
       "authors": ["Terry Yue Zhuo", "Zhuang Li", "Yujin Huang"],
       "year": 2023,
       "arxiv_id": "2301.12868",
       "relevance": "Studies prompt-based adversarial robustness of Codex, finding vulnerability to crafted perturbations."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained code model used for variable renaming perturbation (VarRenamerCB) in the robustness framework."
     },
     {
       "title": "An Exploratory Study on Fine-Tuning Large Language Models for Secure Code Generation",
       "authors": ["Junjie Li", "Fazle Rabbi", "Cheng Cheng"],
       "year": 2024,
       "arxiv_id": "2408.09078",
       "relevance": "Studies fine-tuning LLMs for secure code generation, related to code quality and safety of LLM outputs."
     },
     {
       "title": "Granite Code Models: A Family of Open Foundation Models for Code Intelligence",
       "authors": ["InCoder team"],
       "year": 2022,
       "arxiv_id": "2204.05999",
       "relevance": "Introduces InCoder models for code infilling and synthesis, two variants evaluated in this study."
     }
   ]
 }