ai-research-survey

scan.json (30116B)

---

 {
   "paper": {
     "title": "Prompt Variability Effects On LLM Code Generation",
     "authors": [
       "Andrei Paleyes",
       "Radzim Sendyka",
       "Diana Robinson",
       "Christian Cabrera",
       "Neil D. Lawrence"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2506.10204",
     "doi": "10.48550/arXiv.2506.10204"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "All four LLMs exhibit similar rapid decay in code similarity under keyboard typos augmentation, but are substantially more robust to semantic augmentations (synonyms, paraphrasing). Data contamination significantly reduces measured sensitivity on older LeetCode problems, motivating the use of newer and original tasks. Persona-based evaluation using LLM-simulated user backgrounds (junior engineer, principal engineer, astrophysicist, English teacher) shows qualitative differences in both prompts and generated code, with the largest gaps between non-technical and technical personas.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "An anonymous repository link is provided: https://anonymous.4open.science/r/code-gen-sensitivity-0D19. The abstract states 'we share our code for the benefit of the community.' The custom dataset is also stated to be in this repository."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The custom dataset of 22 programming tasks is available in the repository. LeetCode (Old) is referenced from a public HuggingFace dataset (NyanDoggo/leetcode). LeetCode (New) consists of publicly available recent LeetCode problems."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned in the paper. The paper mentions using NLPaug and specific LLM APIs but does not provide a reproducible environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. The pipeline is described conceptually in Section III.A but there are no explicit commands or scripts referenced for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figures 2, 3, and 4 show 'shaded regions the 95% intervals, calculated from the set of approximately 3400 observations for each rate step' (Section III.E.1)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims like 'Gemini 2.0 Flash is the most resilient to this augmentation method' (Section III.E.1) are based on visual inspection of plots without formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw TSED similarity scores at various augmentation rates (e.g., 'drops below 0.5 TSED after only 10%') but no formal effect sizes such as Cohen's d or standardized differences between models or conditions."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the choice of 5 repetitions per request, 20 LeetCode (New) problems, 22 custom problems, or the overall sample size. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are shown as shaded regions in Figures 2-4, calculated from ~3400 observations per augmentation rate step. Each request was repeated 5 times."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The unaugmented prompt (rate=0) serves as the baseline throughout. Four LLMs are compared against each other. Three augmentation methods provide comparative conditions."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The four evaluated models (GPT-4o mini, Claude 3 Haiku, Gemini 2.0 Flash, Llama 3.3 70B) are all contemporary LLMs at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Three augmentation methods (typos, synonyms, paraphrasing) are compared, and three datasets (LeetCode Old, LeetCode New, Our Dataset) are evaluated separately, effectively ablating the augmentation method and data source components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The quantitative evaluation uses only TSED (Tree Similarity of Edit Distance) as the output similarity metric. BERT Score and Sacre BLEU are used only to validate the paraphrasing augmentation quality, not to evaluate code outputs."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The persona evaluation is analyzed observationally by the authors, but there is no formal human evaluation of the LLM-generated code outputs (no independent human raters, no rating criteria, no inter-rater reliability)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses three separate datasets, none used for any model selection or tuning. LeetCode (New) was specifically chosen from March 2025 to avoid contamination. The custom dataset was created independently."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (LeetCode Old, LeetCode New, Our Dataset) in Figure 4, by augmentation method (typos, synonyms, paraphrasing) in Figures 2-3, and by model across all experiments."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Data contamination on LeetCode (Old) is discussed as a failure mode (Section III.E.2). In the persona evaluation, the English teacher persona received no code for the accounting task — the LLM provided instructions instead (Section IV.C)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The LeetCode (Old) dataset is explicitly reported as unusable for sensitivity evaluation due to data contamination (Section III.B, III.E.2). The English teacher persona failure case is also a negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (1) a synthetic evaluation pipeline (Section III), (2) a persona-based evaluation (Section IV), (3) experimental evidence on multiple LLMs (Section III.E), and (4) open code (anonymous repo link). All are substantiated in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims that 'user background and knowledge have an observable effect on both the prompt and the code generated from it' (Section IV.C), but the persona evaluation uses LLM-simulated personas rather than real humans. The causal claim about human behavior is not justified by this study design. The paper acknowledges 'future work should also explore this experiment using human participants' but the claim stands in the results section."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.B acknowledges that 'any study on LLMs is bound to be missing most recent releases.' The paper specifies exact models tested and presents results as illustrative rather than universal. The abstract calls it 'experimental evidence illustrating utility of our methods.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section VI discusses internal validity (LLM randomness) and external validity (model updates) but does not discuss specific alternative explanations for the findings, such as whether persona-generated prompt differences reflect length/style rather than background knowledge, or whether TSED differences indicate meaningful functional code differences."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'we do not evaluate correctness of generated code, and specifically focus on deviations between the codes generated from the original and altered prompts' (Section III.D), clearly distinguishing what is measured (syntactic code similarity via TSED) from broader claims about code quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-4o mini', 'Claude 3 Haiku', 'Gemini 2.0 Flash', 'Llama 3.3 70B' (Section III.E) — marketing names without specific API versions, snapshot dates, or precise model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes adding 'prefixes and postfixes' to augmented prompts (Section III.E) but does not provide their actual text. Persona descriptions are referenced as being in the repository, not in the paper. One example original prompt is given ('Write a Calculator class...') but augmented prompts and persona instructions are not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 (Section III.E) and repetition count is 5, but no other hyperparameters are reported (max tokens, top-p, system prompts, etc.). These omissions could affect reproducibility."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper makes direct API calls to LLMs for code generation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The augmentation pipeline is described in detail (Section III.A-C): keyboard typos via NLPaug with QWERTY distance, synonyms via NLPaug with WordNet, paraphrasing via Gemini with Sacre BLEU and BERT Score validation. TSED distance computation is described in Section III.D."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' is a dedicated section with subsections on internal validity (VI.A) and external validity (VI.B)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.A discusses specific threats: LLM output randomness even at temperature 0 (mitigated by ~3400 datapoints per step), and small-scale persona evaluation (mitigated by linguistic analytics validation and repetition). Section VI.B discusses rapid model release cycles as an external validity threat."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'we have only considered one-step interactions' and that 'future work should also explore this experiment using human participants' but does not explicitly state what the results do NOT show. These are framed as future directions rather than explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While code and the custom task dataset are referenced in the anonymous repo, there is no explicit mention of raw LLM outputs (the ~3400 responses per augmentation step) being available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are clearly described: LeetCode (Old) from HuggingFace, LeetCode (New) from March 2025 problems, custom dataset of 22 tasks spanning simulations, algorithms, data science, application development, and games (Section III.B). Persona descriptions based on LinkedIn profiles and job descriptions (Section IV.A)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. Data consists of programming task benchmarks and LLM-generated persona outputs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The synthetic pipeline is documented step-by-step in Section III.A: generate n reference outputs, iterate augmentation rates, generate augmented outputs, compute pairwise TSED distances, average. The persona pipeline is documented in Section IV.A-B: define personas, generate prompts (5 iterations), sample one prompt per persona, run in ChatGPT and Claude."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned anywhere in the paper. No acknowledgments section is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with the Department of Computer Science and Technology, University of Cambridge, with email addresses provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The authors are academic researchers not affiliated with any LLM provider being evaluated, which is favorable, but the absence of any funding statement means this cannot be confirmed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four evaluated models. The paper discusses contamination conceptually but does not specify when each model's training data ends."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Data contamination is discussed extensively in Sections III.B and III.E.2. The paper shows empirically that LeetCode (Old) tasks exhibit lower sensitivity due to likely inclusion in training data, citing [12] for detailed contamination analysis."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses contamination by using LeetCode (New) tasks from March 2025 (post-training cutoff) and a custom-created dataset of 22 original tasks. Figure 4 demonstrates the contamination effect on old vs. new datasets."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses LLM-simulated personas, not real human subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All evaluations use LLM APIs and LLM-generated persona outputs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Personas are LLM-simulated character descriptions, not real participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited or studied."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not an experimental study with human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline makes thousands of API calls across four models and multiple conditions, but no inference cost, API spend, or per-example cost is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or hardware details are mentioned despite the significant number of LLM API calls required."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Each request is repeated 5 times at temperature 0, and results are reported with 95% intervals showing variance across runs. The paper notes that 'even setting temperature to 0 does not guarantee fully deterministic results' (Section VI.A) and reports this variability."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section III.E states 'we repeated each request to a LLM 5 times' and 'approximately 3400 datapoints per single augmentation rate step.' Section IV.A states '5 iterations for each persona for each of the three tasks.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The choice of temperature=0, 5 repetitions, and augmentation rate steps is not justified through any search or sensitivity analysis."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper does not select a 'best' configuration — it reports results across all augmentation rates, methods, datasets, and models comprehensively rather than cherry-picking a best result."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose the evaluation pipeline and evaluate it themselves without discussing author-evaluation bias or seeking independent validation of their methodology."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of different sizes and compute costs (Llama 3.3 70B vs GPT-4o mini) are compared without any discussion of compute budget differences or performance-per-compute analysis."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper justifies using TSED over BLEU/BERT Score (Section III.D) but does not discuss whether syntactic tree similarity actually measures meaningful code sensitivity. Two syntactically different programs could be functionally equivalent, a validity gap that is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The paper makes direct API calls to LLMs without any agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is directly addressed: LeetCode (New) uses problems from March 2025 to avoid training data overlap, and the custom dataset was independently created. The paper empirically demonstrates the temporal leakage effect on LeetCode (Old) in Section III.E.2."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup could leak information through the augmented prompts or whether the prompt engineering prefixes/postfixes provide hints not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the three datasets share structural similarities or whether problems within each dataset are independent. The custom tasks span multiple domains but independence is not formally analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No formal leakage detection method (canary strings, membership inference, n-gram overlap analysis) is applied. The paper relies on task recency and originality rather than a concrete detection method."
       }
     }
   },
   "claims": [
     {
       "claim": "All four LLMs exhibit similar sensitivity to keyboard typos augmentation, with code similarity dropping rapidly between 0.0 and 0.6 augmentation rate and plateauing around 0.3 TSED.",
       "evidence": "Figure 2 (left) shows overlapping sensitivity curves for all four models with 95% confidence intervals, aggregated over ~3400 observations per rate step (Section III.E.1).",
       "supported": "strong"
     },
     {
       "claim": "Synonyms and paraphrasing are much weaker augmentation methods than typos, with LLMs showing greater robustness to these semantic changes.",
       "evidence": "Figure 2 (right) shows synonyms only dropping to ~0.5 TSED, and Figure 3 shows similar trends for paraphrasing, compared to ~0.3 for typos (Section III.E.1).",
       "supported": "strong"
     },
     {
       "claim": "Gemini 2.0 Flash is the most robust to synonym augmentation, never dropping below 0.6 in code similarity.",
       "evidence": "Figure 2 (right) shows Gemini 2.0 Flash maintaining higher TSED values than other models under synonym augmentation (Section III.E.1). However, no statistical significance test is reported.",
       "supported": "moderate"
     },
     {
       "claim": "Data contamination causes LLMs to show lower sensitivity on older LeetCode problems compared to newer or original tasks.",
       "evidence": "Figure 4 shows LeetCode (Old) maintaining higher TSED across augmentation rates than LeetCode (New) or the custom dataset, with the custom dataset showing highest sensitivity (Section III.E.2). Supported by reference to [12].",
       "supported": "strong"
     },
     {
       "claim": "User background and knowledge have an observable effect on both the prompt generated and the code produced from it.",
       "evidence": "Section IV.C describes qualitative differences in code generated for different personas (Flask app for principal engineer vs Python classes for junior engineer; no code generated for English teacher on one task). However, this is based on LLM-simulated personas, not real human participants, and is qualitatively analyzed without formal metrics.",
       "supported": "weak"
     },
     {
       "claim": "GPT-4o mini and Gemini 2.0 Flash produce nearly identical code outputs for the same unaltered prompt at temperature 0, while Llama 3.3 and Claude 3 Haiku show higher instability.",
       "evidence": "Section III.E.1 reports ~0.9 similarity for GPT-4o mini and Gemini 2.0 Flash at rate=0, versus higher variance for the other two models. Visible in Figures 2-3.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "LLM personas as proxy for real humans",
       "detail": "The persona evaluation uses LLM-simulated personas (instructing an LLM to adopt a persona and generate prompts) rather than real human participants, but draws conclusions about how human background affects prompt development and code generation. The paper acknowledges this limitation but the claims still stand in the results section."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims about model robustness (e.g., 'Gemini 2.0 Flash is the most resilient') are based on visual inspection of plots without formal statistical tests, despite having sufficient data (~3400 observations per step) to run them."
     },
     {
       "flag": "Single quantitative output metric",
       "detail": "TSED is the only code similarity metric used. No functional correctness evaluation is performed. Syntactically different programs may be functionally equivalent, and the paper does not assess whether TSED differences correspond to meaningful behavioral differences in the code."
     },
     {
       "flag": "Small-scale qualitative persona evaluation",
       "detail": "The persona evaluation uses only 3 tasks, 4 personas, and 2 LLMs with purely qualitative (observational) analysis. No inter-rater reliability, no formal coding scheme, and randomly sampling 1 of 5 generated prompts per persona introduces selection effects."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval, a foundational benchmark for LLM code generation evaluation."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "relevance": "Major benchmark for evaluating LLM ability to resolve real-world software engineering issues."
     },
     {
       "title": "NLPerturbator: Studying the robustness of code LLMs to natural language variations",
       "authors": ["Junkai Chen", "Zhenhao Li", "Xing Hu", "Xin Xia"],
       "year": 2024,
       "arxiv_id": "2406.19783",
       "relevance": "Directly related work on robustness of code LLMs to 18 categories of natural language perturbations."
     },
     {
       "title": "LLM performance for code generation on noisy tasks",
       "authors": ["Radzim Sendyka", "Christian Cabrera", "Andrei Paleyes", "Diana Robinson", "Neil Lawrence"],
       "year": 2025,
       "arxiv_id": "2505.23598",
       "relevance": "Companion paper from same group studying data contamination effects on LLM code generation evaluation."
     },
     {
       "title": "Effectiveness of symmetric metamorphic relations on validating the stability of code generation LLM",
       "authors": ["Patrick Y. P. Chan", "John Keung", "Zhen Yang"],
       "year": 2025,
       "relevance": "Complementary approach using metamorphic testing to validate stability of code-generating LLMs."
     },
     {
       "title": "HumanEvalComm: Benchmarking the communication competence of code generation for LLMs and LLM agent",
       "authors": ["Jie JW Wu", "Fatemeh H. Fard"],
       "year": 2024,
       "relevance": "Benchmark for evaluating LLM communication competence in code generation dialogues."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Rigorous evaluation methodology for LLM-generated code correctness, published at NeurIPS."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces Mostly Basic Programming Problems (MBPP) benchmark for LLM code generation."
     },
     {
       "title": "Quantifying language models' sensitivity to spurious features in prompt design",
       "authors": ["Melanie Sclar", "Yejin Choi", "Yulia Tsvetkov", "Alane Suhr"],
       "year": 2023,
       "relevance": "Studies LLM sensitivity to prompt formatting, closely related to this paper's investigation of prompt variability effects."
     },
     {
       "title": "A natural experiment on LLM data contamination in code generation",
       "authors": ["Mark Roberts", "Hossein Thakur", "Chris Herlihy", "Colin White", "Samuel Dooley"],
       "year": 2023,
       "relevance": "Studies data contamination in LLM code generation, directly relevant to this paper's contamination analysis."
     },
     {
       "title": "Revisiting code similarity evaluation with abstract syntax tree edit distance",
       "authors": ["Yewei Song", "Cedric Lothritz", "Xunzhu Tang", "Tegawendé Bissyandé", "Jacques Klein"],
       "year": 2024,
       "relevance": "Introduces the TSED metric used as the primary code similarity measure in this paper."
     },
     {
       "title": "Out of the Bleu: how should we assess quality of the code generation models?",
       "authors": ["Mikhail Evtikhiev", "Egor Bogomolov", "Yaroslav Sokolov", "Timofey Bryksin"],
       "year": 2023,
       "relevance": "Evaluates NLP code similarity metrics for LLM-generated code, informing the metric choice in this paper."
     }
   ]
 }