ai-research-survey

scan.json (30372B)

---

 {
   "paper": {
     "title": "Are They All Good? Evaluating the Quality of CoTs in LLM-based Code Generation",
     "authors": [
       "Binquan Zhang",
       "Li Zhang",
       "Zhiwen Luo",
       "Yuxin Du",
       "Fang Liu",
       "Song Wang",
       "Lin Shi"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2507.06980",
     "doi": "10.48550/arXiv.2507.06980"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII provides a GitHub link: https://github.com/binquanzzz/CoT-Eval with 'data and scripts online to facilitate replication or future work.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII states data is provided at the GitHub repository. The benchmarks used (CoderEval, SWE-bench) are also publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment setup details are provided in the paper. Only models are named without specifying the runtime environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. The GitHub link is provided but the paper itself contains no instructions for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All main results (pass@1 scores, factor percentages, recall/precision/F1) are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Comparisons between models and between MAD vs single-LLM methods are made by comparing raw percentages without any formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. Results are presented as raw percentages and counts without standardized measures of effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A.3 explicitly justifies the pilot sample: 'we randomly selected 262 samples from 813 failed CoT samples (with a confidence level of 95% and a margin of error of ±5%).'"
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Pass@1 values are single-run results. The taxonomy distribution percentages have no uncertainty measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The MAD framework is compared against single-LLM detection methods (DeepSeek-R1, Gemini-2.0, o1 alone). The repair experiment compares three feedback granularities against a 0% baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All three models used (DeepSeek-R1, Gemini-2.0-Flash-Thinking-Exp-01-21, o1-2024-12-17) are contemporary reasoning models released in late 2024 / early 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The repair experiment (Section V) varies feedback granularity across three levels (simple, error type, detailed error feedback), functioning as an ablation of feedback information. The MAD framework tests different role assignments (DGO, GDO, OGD)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Detection is evaluated with Recall, Precision, and F1-Score (Table III). Code generation uses pass@1. Annotation quality uses Cohen's Kappa."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Four experts with 10+ years of experience manually analyzed CoT quality, supplemented by four additional annotators. Each sample independently annotated by two participants with Cohen's Kappa scores above 0.7."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Code correctness is evaluated using the original benchmark test suites (CoderEval unit tests, SWE-bench patches). The pilot set (262 samples) was used to build the taxonomy, with the remaining 551 annotated separately."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: by factor type (external vs internal), by sub-factor, by model (Table II), by dataset (Figure 7, CoderEval vs SWE-bench-NF), and by pass/fail status (Table I)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Sections III.B and III.C provide detailed failure case examples with figures: Unclear Implementation Details (Figure 2), Missing Dependencies (Figure 3), CoT Inconsistent with Prompt (Figure 4), Implicit Requirement Misinterpretation (Figure 5), Incorrect Planning (Figure 6)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Self-repair results are critically low (0.0% pass@1 for all models on SWE-bench-NF with simple feedback, Table IV). MAD framework shows high recall but very low precision (e.g., MAD OGD: 43.9% recall but 5.49% precision). These limitations are honestly reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (53.60% external factors, 40.10% internal factors, 18.5% correct CoT → wrong code, 11.90% wrong CoT → correct code) are all directly supported by results in Section III.D and Tables I-II."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Most claims are descriptive (factor distributions) rather than strongly causal. The repair experiment's claim that 'more detailed feedback improves repair' is supported by controlled comparison across three feedback levels. The study design is adequate for the claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'LLM-based Code Generation' broadly, but results are only for Python tasks on CoderEval and SWE-bench-NF with 3 specific reasoning models. The threats-to-validity section acknowledges Python focus but the title and abstract do not bound the claims to Python."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats-to-validity section discusses methodological concerns (subjectivity, prompt sensitivity) but does not consider alternative explanations for the observed findings, such as whether the factor distribution would differ with different prompt engineering strategies or whether the CoT quality issues are artifacts of the benchmarks rather than the models."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@1 on test suites and frames it as code correctness — the measurement matches the claim granularity. CoT quality is assessed by expert annotators against explicit criteria. The paper does not overclaim beyond its measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific versions are provided: 'Gemini-2.0-Flash-Thinking-Exp-01-21', 'DeepSeek-R1', and 'OpenAI o1-2024-12-17'. These include snapshot dates or version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The repair feedback templates are provided verbatim in Section V.A. However, the MAD framework prompts (verifier, defender, arbiter roles) are only described in natural language without the actual system prompts. The main experiment uses benchmark inputs directly but the MAD prompts are not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we strictly adhere to the preset parameter configurations' but never specifies what those configurations are. No temperature, top-p, max tokens, or other sampling parameters are reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding (tool use, memory, retry logic) is used. The MAD framework is a multi-agent debate protocol, not agentic scaffolding in the typical sense."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A.2 describes dataset selection: 230 Python tasks from CoderEval, 111 New Features instances from SWE-bench Full. The filtering criteria for SWE-bench-NF (New Features type) is clearly stated. The annotation pipeline (pilot → full annotation) is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.B 'Threats to Validity' provides substantive discussion covering external validity (Python-only, generalizability), internal validity (manual analysis subjectivity), and construct validity (prompt format sensitivity)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: Python-only focus ('specifically focuses on Python code generation tasks'), subjectivity of CoT quality labeling ('different labelers may have varying determinations'), and prompt format sensitivity ('evaluation results may be sensitive to the prompt format'). These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.B states: 'Our empirical study specifically focuses on Python code generation tasks' and notes 'it would be valuable to extend this analysis to other programming languages.' The SWE-bench subset is bounded to New Features only."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII provides a GitHub link (https://github.com/binquanzzz/CoT-Eval) with 'data and scripts online to facilitate replication or future work,' implying raw CoT-code pairs and annotations are available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A describes the collection process: three LLMs given benchmark tasks, producing 1,023 CoT-code pairs (690 from CoderEval, 333 from SWE-bench-NF). Code correctness evaluated per benchmark protocols."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A.3 describes annotators: 'four experts, each with over ten years of experience in Java and Python development' for the pilot; 'one PhD student, two graduate students, and one undergraduate student' with SE research or open-source experience for remaining samples."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: 1,023 total pairs → 813 failed (analyzed for taxonomy) + 210 passing (analyzed for CoT-code relationship). 262 pilot samples → codebook → 551 remaining samples. Each step with counts and criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Funding status is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Beihang University, China and York University, Canada. These are academic institutions with no direct conflict with the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The authors evaluate third-party models (DeepSeek, Google, OpenAI), and no funding relationship with these companies is mentioned or denied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the three models (DeepSeek-R1, Gemini-2.0-Flash-Thinking, o1). This is relevant because CoderEval and SWE-bench are public benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CoderEval or SWE-bench tasks appeared in the training data of the three reasoning models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CoderEval and SWE-bench are public benchmarks published before the models' training. No contamination analysis or discussion is provided, despite the risk that models may have seen benchmark solutions during training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects are studied. The annotators are part of the methodology (labeling CoT quality), not research participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects are studied. Expert annotation of LLM outputs does not constitute human subjects research."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects are studied. Annotator qualifications are described as part of methodology, not as participant demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects are studied."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study with experimental conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study with participants who could drop out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs reported despite using three commercial/expensive reasoning models (o1, DeepSeek-R1, Gemini) across 1,023+ generations plus MAD experiments (3 models per detection) plus repair experiments."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget or API costs are stated. The MAD framework is acknowledged to have 'significant computational overhead' (Section IV.B) but this is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or multiple runs. All results appear to be from single runs per model."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V.A states 'For each problem requiring repair, we generated one code solution.' The pass@1 metric implies single-attempt evaluation. The number of runs is implicitly clear."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section VI.B mentions 'small-scale preliminary testing, experimenting with different prompts' but does not report how many configurations were tried or the search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions selecting prompts that 'yield the most consistent and optimal performance across various models' but provides no details on the selection process or what configurations were considered."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The MAD framework is the authors' proposed approach, compared against single-LLM methods. No acknowledgment of potential bias in evaluating their own method, such as whether the MAD prompt design was tuned to perform well."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The MAD framework uses 3 models per detection (vs 1 for baselines) — roughly 3x compute. Section IV.B mentions 'significant computational overhead' but does not quantify the cost-performance tradeoff."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CoderEval and SWE-bench-NF actually measure what the paper claims to evaluate (CoT quality in code generation). The paper uses pass@1 as the ultimate criterion without questioning whether test-case passing equates to code quality."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are directly prompted with benchmark tasks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. CoderEval (2024) and SWE-bench (2023) were published before the models used, and solutions may appear in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The benchmarks provide docstrings and context that may partially overlap with training data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples share structural similarities with training data or with each other."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods applied. No canary strings, membership inference, or decontamination."
       }
     }
   },
   "claims": [
     {
       "claim": "Among the factors affecting CoT quality, external factors account for 53.60% and internal factors make up 40.10%. External factors are primarily unclear requirements and lack of contextual information; internal factors are mainly misunderstanding of instructions.",
       "evidence": "Section III.D reports distributions based on manual annotation of 813 failed CoT samples by expert annotators with Cohen's Kappa ≥0.7. Table II provides per-model breakdowns.",
       "supported": "strong"
     },
     {
       "claim": "Despite CoT being correct, 18.5% of the generated code still contains errors, primarily due to LLMs failing to follow their own CoT steps.",
       "evidence": "Section III.D: analysis of 210 passing code samples and their associated CoTs. Also Finding 3 states 'even with a correct CoT, there remains an 18.5% probability of generating erroneous code.'",
       "supported": "moderate"
     },
     {
       "claim": "Even when the code is correct, there is an 11.9% chance that the CoT contains errors.",
       "evidence": "Section III.D and Table I: among 210 passing code samples, 25 had incorrect CoTs (25/210 = 11.9%).",
       "supported": "moderate"
     },
     {
       "claim": "The MAD framework significantly outperforms single-model approaches for detecting low-quality CoTs, particularly in Recall and F1-score.",
       "evidence": "Table III: MAD DGO achieves 42.5% recall vs 8.75% for DeepSeek-R1 alone on CoderEval, and 46.36% vs 23.64% on SWE-bench-NF. However, precision is notably low (12.50% for MAD DGO on CoderEval).",
       "supported": "moderate"
     },
     {
       "claim": "The level of detail in feedback positively impacts LLMs' self-repair performance for faulty CoTs.",
       "evidence": "Table IV: Detailed error feedback yields the highest pass@1 across models (e.g., Gemini-2.0: 5.2% → 5.7% → 7.4% on CoderEval). However, absolute improvements are very small and no statistical tests confirm significance.",
       "supported": "weak"
     },
     {
       "claim": "76.4% of CoTs generated by LLMs have quality issues.",
       "evidence": "Section III.D and Table I: 782/1023 CoT-code pairs had quality issues. The proportion is higher for SWE-bench-NF (82.3%) than CoderEval (73.6%).",
       "supported": "moderate"
     }
   ],
   "key_findings": "This empirical study of 1,023 CoT-code pairs from three reasoning models (DeepSeek-R1, o1, Gemini-2.0-Flash-Thinking) finds that 76.4% of generated CoTs have quality issues, with external factors (unclear requirements, missing context) accounting for 53.6% and internal factors (requirement misunderstanding, incorrect planning) 40.1%. A key finding is the disconnect between CoT and code quality: correct CoTs still produce erroneous code 18.5% of the time, while 11.9% of correct code comes from flawed CoTs, indicating LLMs don't strictly follow their reasoning chains. A Multi-Agent Debate framework improves faulty CoT detection recall (up to 48.5%) but suffers from very low precision (as low as 4.85%), and self-repair capabilities remain critically limited.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All main results (pass@1, factor distributions, recall/precision/F1) are reported as point estimates without confidence intervals, error bars, or statistical significance tests. Differences between models and methods are presented as meaningful without any formal testing."
     },
     {
       "flag": "Complete absence of contamination analysis",
       "detail": "CoderEval (2024) and SWE-bench (2023) are public benchmarks that may appear in the training data of the three reasoning models. No training cutoff dates are stated and no contamination analysis is performed, yet pass@1 scores are interpreted at face value."
     },
     {
       "flag": "Single-run results without replication",
       "detail": "Pass@1 is computed from single-run generation with no repeated trials or seed sensitivity analysis. Given the stochastic nature of LLM generation, single-run results may not be stable."
     },
     {
       "flag": "Cohen's Kappa overclaimed",
       "detail": "Section III.A.3 reports Cohen's Kappa of 82% as 'indicating perfect agreement.' By the standard Landis-Koch interpretation, 0.82 is at the bottom of 'almost perfect' (0.81-1.00), not 'perfect.'"
     },
     {
       "flag": "Hyperparameters not specified",
       "detail": "The paper uses three reasoning models with 'preset parameter configurations' that are never specified. Temperature, top-p, and other sampling parameters significantly affect output and are not reported."
     },
     {
       "flag": "MAD precision critically low",
       "detail": "While the MAD framework's recall is emphasized, its precision is extremely low (as low as 4.85% for MAD OGD on SWE-bench-NF), meaning it produces overwhelming false positives. The paper's framing emphasizes the recall improvement without adequately addressing the practical uselessness of such low precision."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Foundational benchmark for evaluating LLMs on real-world software engineering tasks, used as one of two evaluation datasets in this study."
     },
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Proposes EvalPlus framework for rigorous LLM code generation evaluation, addressing test adequacy in benchmarks."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma"],
       "year": 2022,
       "relevance": "Foundational paper introducing CoT prompting that this study directly evaluates for quality in code generation."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Open-source reasoning model evaluated in this study, relevant to LLM capability assessment."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and code generation evaluation methodology including pass@k metrics widely adopted in the field."
     },
     {
       "title": "Structured chain-of-thought prompting for code generation",
       "authors": ["Jia Li", "Ge Li", "Yongmin Li", "Zhi Jin"],
       "year": 2023,
       "relevance": "Proposes structured CoT for code generation, directly related to the CoT quality evaluation in this study."
     },
     {
       "title": "Bugs in large language models generated code: An empirical study",
       "authors": ["Florian Tambon", "Arghavan Moradi Dakhel", "Amin Nikanjam", "Foutse Khomh"],
       "year": 2024,
       "arxiv_id": "2403.08937",
       "relevance": "Classifies defective code generated by LLMs using CoderEval, providing an error taxonomy for LLM code quality."
     },
     {
       "title": "Chain-of-thought reasoning in the wild is not always faithful",
       "authors": ["Iván Arcuschin", "Jett Janiak", "Robert Krzyzanowski"],
       "year": 2025,
       "arxiv_id": "2503.08679",
       "relevance": "Directly relevant to the CoT-code disconnect finding: studies whether CoT faithfully represents model reasoning."
     },
     {
       "title": "Refining chatgpt-generated code: Characterizing and mitigating code quality issues",
       "authors": ["Yue Liu", "Thanh Le-Cong", "Ratnadira Widyasari"],
       "year": 2024,
       "relevance": "Systematic evaluation of ChatGPT code reliability including quality attributes beyond correctness."
     },
     {
       "title": "ClarifyGPT: A framework for enhancing LLM-based code generation via requirements clarification",
       "authors": ["Fangwen Mu", "Lin Shi", "Song Wang"],
       "year": 2024,
       "doi": "10.1145/3660810",
       "relevance": "Addresses requirement ambiguity in LLM code generation — directly related to the external factors (unclear requirements) identified in this study."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "relevance": "Alternative reasoning framework to CoT, relevant to understanding LLM reasoning strategies for code generation."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Evaluates LLM-based approaches to software engineering tasks including code generation, relevant to agentic AI evaluation."
     },
     {
       "title": "Code repair with LLMs gives an exploration-exploitation tradeoff",
       "authors": ["Hao Tang", "Keya Hu", "Jin Zhou"],
       "year": 2024,
       "relevance": "Studies LLM self-repair capabilities for code, directly relevant to the self-repair investigation in this study."
     },
     {
       "title": "Encouraging divergent thinking in large language models through multi-agent debate",
       "authors": ["Tian Liang", "Zhiwei He", "Wenxiang Jiao"],
       "year": 2023,
       "arxiv_id": "2305.19118",
       "relevance": "Introduces the Multi-Agent Debate framework adopted in this study for detecting low-quality CoTs."
     }
   ]
 }