ai-research-survey

scan.json (29561B)

---

 {
   "paper": {
     "title": "Identifying Inaccurate Descriptions in LLM-generated Code Comments via Test Execution",
     "authors": ["Sungmin Kang", "Louis Milliken", "Shin Yoo"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2406.14836",
     "doi": "10.48550/arXiv.2406.14836"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "Even the best-performing LLM (GPT-4) generates factually inaccurate code comments in roughly 20% of cases. Nine existing code-comment consistency detection techniques showed no statistically significant relationship with comment factual accuracy. The proposed 'document testing' concept — generating tests from comments via LLMs and observing pass/fail — yields a robust statistical relationship with comment accuracy (p < 10⁻⁹, ROC-AUC 0.67), with failing tests being more predictive of inaccuracies than passing tests.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided in the paper. Supplementary material is mentioned for prompts but no code release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses public Defects4J methods, but the 540 labeled comments, accuracy labels, and generated test results are not released. No dataset download link is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency versions are described. Only the model version gpt-3.5-turbo-0125 and the Defects4J command line interface are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The pipeline is described conceptually (Figure 3) but without runnable scripts or commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported throughout: Figure 4a (pass rate by accuracy), Figure 5 (ROC-AUC and AP), Figure 6 (binned inaccuracy rate), Figure 7 (ablation), and Figure 8 (w sensitivity)."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Welch's t-test and Point-Biserial correlation are used systematically. Table 1 reports p-values for all 9 baselines. Section 6.1 reports p=0.002 for pass rate difference and p < 10⁻⁹ for the correctness estimator."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "ROC-AUC (0.67) and Average Precision are reported as effect size measures. Pass rate differences (~0.55 vs ~0.35) with CIs are shown. Practical thresholds are given: 'rejecting comments with score < 0.8 would remove 46% of inaccurate comments while retaining 72% of correct ones.'"
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "180 methods were sampled from Defects4J but no justification is given for this specific number. No power analysis is discussed. The 141 unambiguously labeled comments used for evaluation are a subset determined by labeling outcome, not design."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 states 'we repeated experiments five times to verify that our approach works consistently.' Results are reported as means with 95% confidence intervals across the five runs (Figures 5, 7, 8)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Nine baselines are evaluated: four code-comment consistency techniques (DocChecker, Deep-JIT, GPT-3-NoCoT, GPT-3-CoT), four similarity measures (BLEU, SentenceBERT, CodeT5, CodeBERT), and one LLM inaccuracy detector (CID). Results in Table 1 and Figure 5."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include DocChecker (2024), Li and Shin/GPT-3-NoCoT (2024), CID (2024), Deep-JIT (2021), CodeT5 (2021), CodeBERT (2020). The most relevant consistency detection baselines are contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 6.2, Figure 7) is an explicit prompt ablation study, removing components incrementally: comment/signature only → + class name → + constructor → + example tests → + two-stage prompting. Also compares EvoSuite vs human tests."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: ROC-AUC, Average Precision (AP), Welch's t-test p-values, Point-Biserial correlation, and test pass rate. Section 5.2 justifies the choice of each metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 6.4) provides a qualitative evaluation where the authors manually analyze the system's outputs — examining specific test cases that succeeded (Figure 9) and failed (Figures 10, 11) to assess strengths and limitations."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 141 labeled comments are used for both evaluation (RQ1-2 with w=100) and parameter analysis (RQ3 exploring w on the same data). No train/dev/test split is used. The w=100 parameter choice for RQ1-2 is not justified from a separate validation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by LLM (Figure 1: StarCoder, GPT-3, GPT-4), by accuracy type (Figure 4a: accurate, inaccurate, behaviorally inaccurate), and by error taxonomy (Figure 2: four error categories with counts)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 explicitly discusses failure cases: execution environment failures (Figure 10, test fails due to missing CSV file) and LLM-hallucinated properties (Figure 11, test based on fabricated property not in the comment)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows all 9 baseline techniques fail to detect inaccurate comments (a major negative result). Section 6.4 reports failure modes of the proposed approach. CodeBERT's spurious ROC-AUC is investigated and debunked (Section 6.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims: (1) ~1/5 GPT-4 comments inaccurate → supported by Figure 1. (2) Nine techniques have no statistically significant relationship → supported by Table 1. (3) Document testing has robust statistical relationship → supported by Section 6.1 (p < 10⁻⁹, ROC-AUC 0.67)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main claims are correlational ('statistical relationship'). Causal claims in the ablation study (RQ2, 'each element helps improve') are justified through controlled single-variable manipulation in Figure 7."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'LLM-generated Code Comments' broadly, but results are only on Java methods from Defects4J with GPT-3.5-turbo. Section 7.1 acknowledges the limitation ('further research is required to tell whether these principles would work for other languages and projects'), but the title and abstract frame the contribution more broadly than the evidence supports."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 discusses LLM randomness (mitigated by 5 runs), training data contamination risk (argued not to invalidate results). Section 6.1 investigates CodeBERT's spurious performance. Section 6.4 discusses failure modes where the core assumption breaks down."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly formalizes the proxy relationship: test pass rate as a proxy for comment accuracy, with a full Bayesian derivation (Section 4.1, Equations 1-13). Limitations of this proxy are discussed in Sections 6.4 and 7 (hallucinated properties, execution environment issues)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "GPT-3 is versioned as 'gpt-3.5-turbo-0125' (Section 5.2). However, GPT-4 is referenced only as 'GPT-4 [19]' without a version or snapshot date, and StarCoder is referenced only by name without a specific version. Both are used in the accuracy evaluation (Section 2.1, Figure 1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The specific prompt is provided in our supplementary material' (Section 2.1) and 'The full prompt is provided in the supplementary material' (Section 4.2.2). Prompts are provided, though in supplementary material rather than the main paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported for any model used in the experiments. Only the model name/version and number of runs (5) are stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The pipeline (Figure 3) is a sequential information retrieval → two-stage prompting → test execution workflow, not an agentic system with tools, memory, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents the sampling procedure: 180 public methods from Defects4J fixed files, selected for longest comments, generating 540 comments across 3 LLMs. Labeling procedure described with two annotators and 87% agreement. Section 5.2 notes 141 unambiguous labels used."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Discussion' includes Section 7.1 'Threats to Validity' with substantive discussion of internal and external threats. Section 7.2 'Future Work' also discusses current limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 discusses specific threats: LLM output randomness mitigated by 5 runs, GPT training data potentially containing subject code (specific to their choice of Defects4J), results limited to Java code from Defects4J benchmark."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 explicitly states: 'the experimental results we present were done on Java code from the widely-used Defects4J benchmark; further research is required to tell whether these principles would work for other languages and projects.' Section 7.2 identifies specific expansions needed."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 540 labeled comments, accuracy labels, generated tests, and test execution results are not released. No data download link or archive is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes data collection in detail: sampling criteria (public methods from Defects4J fixed files, longest comments), three LLMs used, labeling procedure (first author labels, second author independently labels subset, 87% agreement, discussion to resolve disagreements)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses code from the standard public Defects4J benchmark. Author labeling is part of the methodology, not human subject recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: 180 methods from Defects4J → 540 comments (3 LLMs × 180) → manual labeling → 141 unambiguous GPT-3 labels for experiments. The document testing pipeline (Figure 3) documents information retrieval → prompting → test execution stages."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from KAIST (Daejeon, South Korea), clearly stated in the author block. They evaluate third-party products (GPT, StarCoder) with no affiliation to OpenAI or BigCode."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed. KAIST authors evaluating third-party tools suggests no obvious conflict, but the absence of any funding disclosure means this cannot be confirmed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit training data cutoff dates are stated for GPT-3.5-turbo, GPT-4, or StarCoder. Section 7.1 acknowledges 'the training data of the GPT family of LLMs is unknown' but does not state any cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 directly discusses this: 'As the training data of the GPT family of LLMs is unknown, there is also the risk that the LLM had learned the subject code. However, this did not translate into the LLM correctly predicting which comments were accurate on its own; only with document testing could we find a reliable predictor.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Defects4J has been publicly available since 2014, well before GPT model training. Section 7.1 acknowledges the contamination risk and argues it does not invalidate the document testing results, since the LLM alone cannot predict comment accuracy without testing."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM-generated comments on code from Defects4J. Author labeling is methodological, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes code and LLM-generated comments."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or wall-clock time are reported despite the approach requiring multiple LLM calls per comment (property extraction + test generation) plus test compilation and execution."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The approach requires 5 runs × 141 comments × multiple LLM calls each, plus EvoSuite runs for one condition, but none of this is quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2: 'we repeated experiments five times to verify that our approach works consistently.' Results are reported with 95% CIs across runs, demonstrating sensitivity to LLM randomness (analogous to seed sensitivity)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 explicitly states: 'we repeated experiments five times.' The top three ablation experiments in Figure 7 were performed once (noted in caption)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "RQ3 explores the w parameter across 201 values using a logarithmic schedule (Section 6.3), but the selection of w=100 for RQ1 and RQ2 is not justified through a principled search on a separate validation set. No search budget for other design choices is stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "w=100 is used in RQ1 and RQ2 before its sensitivity is explored in RQ3. While RQ3 confirms w>10 works well, the specific choice of w=100 is not justified through validation set selection or principled methodology."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports p-values for 9 baseline comparisons, and Section 6.1 reports additional significance tests. No correction for multiple comparisons (Bonferroni, Holm, etc.) is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose and evaluate their own document testing technique against baselines. While they retrained Deep-JIT to confirm similar accuracy to the original paper, they do not discuss the general bias of evaluating their own system."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Document testing requires multiple LLM calls plus test compilation and execution per comment, which is substantially more compute than any baseline (which require a single forward pass). This compute difference is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 extensively argues that their factual accuracy definition is more meaningful than BLEU or Likert-scale evaluations. They justify their labeling criteria with concrete examples (Section 3) and distinguish it from subjective 'accuracy' in prior work."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding comparison is involved. The approach uses a fixed pipeline; no model-to-model comparison through different scaffolds occurs."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J code has been publicly available since 2014, predating all LLM training. While Section 7.1 discusses the general contamination risk, no temporal analysis is performed to assess whether specific methods or their tests appeared in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 explicitly withholds the method body from test generation prompts: 'only the signature is provided because the aim is to evaluate the comment; if the code itself is exposed, the LLM may be more influenced by the code than the comment.' This is a deliberate leakage prevention measure."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Methods are sampled from multiple Defects4J projects, but no analysis of whether methods from the same project share characteristics that could inflate results. Non-independence of examples from the same class or project is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, n-gram overlap, decontamination pipeline) is applied. The code withholding in Section 4.2.1 is a design choice for the pipeline, not a data contamination detection method."
       }
     }
   },
   "claims": [
     {
       "claim": "Even the best-performing LLM (GPT-4) generates factually inaccurate code comments in roughly one fifth of cases.",
       "evidence": "Manual inspection of 540 LLM-generated comments across StarCoder, GPT-3, and GPT-4. Figure 1 shows GPT-4 has ~20% inaccurate comments, GPT-3 ~33%, StarCoder ~50%. Inter-rater agreement at 87% (Section 2.1).",
       "supported": "strong"
     },
     {
       "claim": "Nine existing techniques (code-comment consistency detectors, similarity metrics, LLM self-inspection) show no statistically significant relationship with comment factual accuracy.",
       "evidence": "Table 1 reports Welch's t-test and Point-Biserial correlation p-values for all 9 baselines on 141 unambiguous GPT-3 labels. All p-values exceed 0.05 (Section 2.2).",
       "supported": "strong"
     },
     {
       "claim": "Document testing has a robust statistical relationship with comment accuracy, unlike all baselines.",
       "evidence": "Section 6.1: test pass rate difference between accurate and inaccurate comments (p=0.002), correctness estimator p < 10⁻⁹ (Welch's t-test), p < 10⁻¹¹ (Point-Biserial). ROC-AUC 0.67. Figure 5 shows document testing outperforms all baselines on both ROC-AUC and AP with non-overlapping 95% CIs.",
       "supported": "strong"
     },
     {
       "claim": "Each component of the helper information (class name, constructors, example tests, two-stage prompting) improves document testing performance.",
       "evidence": "RQ2 (Section 6.2, Figure 7) ablation study shows incremental improvement in both executable test proportion and ROC-AUC as components are added.",
       "supported": "moderate"
     },
     {
       "claim": "Failing tests are more predictive of inaccurate comments than passing tests (optimal w > 10).",
       "evidence": "RQ3 (Section 6.3, Figure 8) shows ROC-AUC peaks when w > 10, meaning failing tests are weighted more heavily. Explained by labeling procedure: one inaccurate statement makes the whole comment inaccurate.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Small sample size for main evaluation",
       "detail": "Only 141 unambiguously labeled GPT-3 comments are used for the main evaluation (RQ1-4). The 540 comments span 3 LLMs, but only GPT-3's unambiguous subset is evaluated with document testing. The StarCoder validation is mentioned but not systematically reported."
     },
     {
       "flag": "Single primary annotator",
       "detail": "The first author labeled all 540 comments. The second author independently labeled only a 'subset' (proportion unspecified). At 87% agreement, 13% of labels are potentially contested. The final discussed labels are used, but the process relies heavily on one annotator."
     },
     {
       "flag": "Modest discriminative ability",
       "detail": "ROC-AUC of 0.67 is statistically significant and better than baselines (all near 0.5), but indicates limited practical discriminative ability. The paper frames this as strong but a random classifier achieves 0.5."
     },
     {
       "flag": "Parameter selection on evaluation data",
       "detail": "w=100 is used in RQ1-2 before RQ3 explores the parameter space on the same 141 examples. This circular use of the evaluation data could inflate reported performance."
     }
   ],
   "cited_papers": [
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM evaluated for comment generation accuracy, directly relevant to LLM code generation capabilities."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "State-of-the-art LLM evaluated for comment generation; relevant to LLM capability assessment."
     },
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["Angela Fan", "Beliz Gokkaya", "Mark Harman"],
       "year": 2023,
       "arxiv_id": "2310.03533",
       "relevance": "Survey of LLMs for SE tasks including code generation and documentation."
     },
     {
       "title": "Large language models are few-shot summarizers: Multi-intent comment generation via in-context learning",
       "authors": ["Mingyang Geng", "Shangwen Wang", "Dezun Dong"],
       "year": 2024,
       "relevance": "LLM-based code comment generation achieving state-of-the-art; relevant to automated documentation quality."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained code model used as baseline for code-comment consistency, relevant to code understanding capabilities."
     },
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": ["Yue Wang", "Weishi Wang", "Shafiq Joty"],
       "year": 2021,
       "arxiv_id": "2109.00859",
       "relevance": "Pre-trained code model used as baseline; relevant to code generation and understanding."
     },
     {
       "title": "Large language models are few-shot testers: Exploring LLM-based general bug reproduction",
       "authors": ["Sungmin Kang", "Juyeon Yoon", "Shin Yoo"],
       "year": 2023,
       "relevance": "LLM-based test generation for bug reproduction; foundational to the document testing pipeline."
     },
     {
       "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Self-consistency technique for estimating LLM correctness; alternative approach to hallucination detection."
     },
     {
       "title": "Enhancing Trust in LLM-Generated Code Summaries with Calibrated Confidence Scores",
       "authors": ["Yuvraj Virk", "Premkumar Devanbu", "Toufique Ahmed"],
       "year": 2024,
       "arxiv_id": "2404.19318",
       "relevance": "Directly related work on trusting LLM-generated code documentation using token probabilities."
     },
     {
       "title": "Chain of Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Chain-of-thought prompting technique used as a baseline variant (GPT-3-CoT) in experiments."
     },
     {
       "title": "Whole Test Suite Generation",
       "authors": ["Gordon Fraser", "Andrea Arcuri"],
       "year": 2013,
       "relevance": "EvoSuite automated test generation tool used as an alternative to human tests in the document testing pipeline."
     },
     {
       "title": "Mutation-based consistency testing for evaluating the code understanding capability of LLMs",
       "authors": ["Ziyu Li", "Donghwan Shin"],
       "year": 2024,
       "arxiv_id": "2401.05940",
       "relevance": "LLM code understanding evaluation via mutation-based consistency; baseline technique in this study."
     }
   ]
 }