ai-research-survey

scan.json (30102B)

---

 {
   "paper": {
     "title": "HumanEvalComm: Benchmarking the Communication Competence of Code Generation for LLMs and LLM Agent",
     "authors": ["Jie JW Wu", "Fatemeh H. Fard"],
     "year": 2024,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2406.00215",
     "doi": "10.1145/3715109"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a public GitHub repository: 'Our benchmark and replication package are made public at https://github.com/jie-jw-wu/human-eval-comm' (Section 1, Section 9)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The HumanEvalComm benchmark dataset is publicly released at the same GitHub repository. The paper states 'Our benchmark and full code are publicly available.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes hardware (Intel i7-6700K, Tesla V100-SXM2-16GB) and Python 3.12, and lists HuggingFace model names, but does not provide a requirements.txt, Dockerfile, or detailed dependency list sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper calls its release a 'replication package' but does not contain step-by-step reproduction instructions within the paper itself. No 'Reproducing Results' section or specific commands are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Tables 3-5 report point estimates without confidence intervals or error bars. RQ4 (Table 7-8) reports variance from 5 runs, but the primary results lack uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Independent two-sample t-tests with p-values are reported throughout Tables 4-5 at α = 0.01, 0.05, 0.1 significance levels (Section 3.5)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute differences are reported with baseline context throughout (e.g., 'Pass@1 drop by 35%∼52%', 'increases Communication Rate by an absolute 58%'). Tables show both HumanEval and HumanEvalComm values, enabling effect size assessment."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "For the manual evaluation (RQ3), the paper explicitly uses a sample size calculator: 'we set Confidence Level as 95%, Margin of Error as 5%, and Population Proportion as 50%' to determine 60 samples per model (Section 4.3)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Tables 3-5) are reported as single-run point estimates without variance or standard deviation across runs. Only the RQ4 investigation (Tables 7-8) reports variance from 5 repeated runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five Code LLMs (ChatGPT, CodeLlama, CodeQwen1.5 Chat, DeepSeek Coder, DeepSeek Chat) serve as baselines for evaluating Okanagan. AgentCoder is also tested as an external LLM agent baseline (Section 5.4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include CodeLlama, DeepSeek Coder, CodeQwen1.5 Chat, and ChatGPT 3.5, all released in 2023-2024 and contemporary to the study. AgentCoder (2023) is also included."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Okanagan has a multi-round structure with specific design choices (3 rounds, reflection pattern), but no systematic ablation removes individual components to measure their contribution. Switching base models (ChatGPT vs DeepSeek Coder) is tested but is not a proper ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation metrics are used: Communication Rate, Good Question Rate, Pass@1, and Test Pass Rate (Section 2.2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Section 4.3) involves 6 graduate students manually evaluating question quality and answer quality from models' responses, with inter-rater reliability measured via Cohen's Kappa (Table 6)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The full HumanEvalComm benchmark (762 problems) is used for both prompt development and evaluation. The authors note they 'optimized the prompt for the LLM-based evaluator several times and checked the results manually' on the same data used for reporting."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed breakdowns by clarification category (1a, 1c, 1p, 2ac, 2cp, 2ap) are provided in Tables 4 and 5, with per-model results for each category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 provides example cases including a failure case (Table 12, CodeLlama on problem 45 where inconsistency was not detected). Okanagan's tendency to ask unnecessary questions is discussed as a limitation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Okanagan achieves lower Pass@1 on original HumanEval than ChatGPT alone (Section 4.2), AgentCoder had 0% communication rate (Section 5.4), and Okanagan with DeepSeek Coder as base underperforms vs ChatGPT base (Section 4.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (>60% code generation, 35-52% Pass@1 drop, 58% Communication Rate increase, 38% Good Question Rate increase for Okanagan) are supported by Table 3 data. However, there is a discrepancy with body text numbers (Section 1 states '59% and 5%' instead of '58% and 38%')."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The claim that Okanagan 'boosts' performance uses causal language. The study design (same base model ChatGPT, same data, only difference is agent structure) provides a controlled comparison adequate for this causal claim. Testing with an alternative base model (DeepSeek Coder) adds robustness."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (External Validity) explicitly states: 'we cannot make a sound claim regarding the communication capability of the models on another dataset.' The paper is tested on Python problems only and the scope is appropriately bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses how the LLM-based evaluator's False Recovery Rate could inflate pass rates for some models. Section 4.1 offers hypotheses for differences between models (generative nature, training data composition). Section 7 addresses internal validity threats."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper formally defines 'communication capability' (Section 1) and directly measures it via Communication Rate and Good Question Rate. Pass@1 and Test Pass Rate directly measure code correctness. The metrics match the granularity of claims without overreaching into broader framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'gpt-3.5-turbo-0125' for ChatGPT, and HuggingFace model names 'deepseek-coder-6.7b-instruct', 'deepseek-llm-7b-chat', 'CodeQwen1.5-7B-Chat', 'CodeLlama-13b-Instruct-hf' (Section 3.5)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Tables 14-16 (Appendix), the LLM-based evaluator prompt in Section 3.2, and prompt variants in Table 15. Actual prompt templates with placeholder variables are shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature=1.0, n=1 for ChatGPT API, max_new_tokens=512 for open-source models, default values for other parameters are reported (Section 3.5, Section 7)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Okanagan's 3-round structure is described in detail (Section 3.4): Round 1 generates code, Round 2 asks questions, Round 3 regenerates with reflection. Figure 3 provides visual illustration. Parameters (number of agents, rounds, actions, thinking pattern) are enumerated."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents the benchmark construction process in detail: taxonomy of clarification types, guidelines for modification, annotator qualifications, discussion process, time estimates (100 hours initial + 30 hours review), and statistics (Table 1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 'Limitations' provides a dedicated subsection with four specific limitations. Section 7 'Threats to Validity' provides additional discussion organized by construct, internal, and external validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: subjectivity in manual modifications, different engineers approaching problems differently, LLM-based evaluator tending to mark more 'Good' questions than human annotators, False Recovery Rates for specific models (30-40%), and Okanagan asking unnecessary questions."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 External Validity explicitly states: 'we cannot make a sound claim regarding the communication capability of the models on another dataset.' The paper notes it only tests Python problems and a limited set of models."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark, code, and replication package are publicly available at the GitHub repository. The paper states: 'we release our complete code and dataset' (Section 7)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the benchmark construction in detail: source (HumanEval), modification taxonomy (Ambiguity, Inconsistency, Incompleteness), annotator process (two software engineers, disagreement resolution), and time estimates."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "For manual evaluation: 'We recruited six graduate students at the University of British Columbia' from 'the software engineering lab in the computer science department, all having at least two years of professional software development background' (Section 4.3). For benchmark creation: annotator qualifications are described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from HumanEval to HumanEvalComm is documented with counts (Table 1: 164 per single type, varying for combinations, 762 total). The evaluation pipeline is shown in Figure 2 with clear steps. Manual evaluation uses hash-function sampling of 60 problems per model."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 10: 'This research is supported by a grant from Natural Sciences and Engineering Research Council of Canada RGPIN-2019-05175.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are affiliated with University of British Columbia, Kelowna, Canada. They evaluate models from external companies (OpenAI, Meta, DeepSeek, Qwen) with no affiliation to those companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSERC is the Canadian government research funding agency with no financial interest in the evaluation outcome of any specific LLM."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models (ChatGPT, CodeLlama, DeepSeek Coder, DeepSeek Chat, CodeQwen1.5)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEvalComm is based on HumanEval (published 2021). All evaluated models were trained after 2021 and may have seen the original problems. While the modifications are new, the base problems and test cases overlap. This is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval has been widely available online since 2021. The paper does not discuss the contamination risk from models having seen the original HumanEval problems during training, even though modifications partially mitigate this."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the manual evaluation study with 6 graduate students."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the manual evaluation involving 6 graduate students."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "The 6 students are described as 'from the software engineering lab in the computer science department, all having at least two years of professional software development background' (Section 4.3)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Participants are from 'the software engineering lab' with 'at least two years of professional software development background' — these serve as implicit inclusion criteria (Section 4.3)."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a manual evaluation task, not an experimental study with treatment conditions. Randomization of participants to conditions does not apply."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "'To avoid introducing bias, we hide the actual names of the 6 models and used model 1 to model 6 instead of actual model names' (Section 4.3)."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on whether all 6 students completed all their assigned evaluations or if any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "API costs are reported: 'the total cost in using ChatGPT 3.5 is less than 15 USD' and 'the cost for running AgentCoder alone in HumanEvalComm is about 36 USD.' Per-token pricing is stated (Section 3.5)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified (Intel i7-6700K, 32GB RAM for API models; Xeon Gold 6130, 44GB RAM, 4x Tesla V100 for open-source models). Total API cost <15 USD. Runtime 'less than half an hour' for Okanagan/ChatGPT (Section 3.5)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Tables 3-5) are single-run results. Only the RQ4 investigation (Table 7-8) repeats experiments 5 times. The primary evaluation does not report seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "For the main results (RQ1, RQ2), the number of runs is not explicitly stated. RQ4 states '5 times' for the sensitivity analysis, but main experiments lack this information."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Default parameters are used throughout, but the choice of defaults is not justified with a search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses default/standard configurations and reports results for all tested configurations. RQ4 (Section 4.4) tests prompt variants and hyperparameter variations, presenting all results (Tables 7-9) rather than cherry-picking."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many t-tests are performed across 6 clarification categories × 6 models × 2 metrics, but no correction for multiple comparisons (Bonferroni, Holm, etc.) is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose and evaluate Okanagan against baselines without acknowledging the bias of evaluating their own system. No independent evaluation or discussion of author-evaluation bias is present."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Okanagan uses 3 LLM calls per problem vs 1-2 for base models, but performance is not compared at matched compute budgets. The 3x compute overhead is not analyzed against the performance gain."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses why automated modification fails (doesn't guarantee triggering clarifying questions). RQ3 validates the evaluation metrics against human judgment. Section 2 justifies the taxonomy of clarification types with RE literature."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests Okanagan with two different base models (ChatGPT and DeepSeek Coder) to separate scaffold effects from model effects (Section 4.2), though the analysis reveals prompt compatibility issues."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and is widely available online. All evaluated models were trained after 2021 and may have memorized solutions. The modifications mitigate but do not eliminate this risk, and it is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (providing function signatures, test cases) leaks information about the correct solution beyond what would be available in realistic usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between the 762 modified problems, which are derived from only 164 original HumanEval problems — performance on modifications of the same problem may be correlated."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. No canary strings, membership inference, or decontamination analysis is performed."
       }
     }
   },
   "claims": [
     {
       "claim": "More than 60% of responses from Code LLMs still generate code rather than ask questions when problem descriptions are modified to be ambiguous, inconsistent, or incomplete.",
       "evidence": "Table 3 shows communication rates: ChatGPT 14.21%, CodeLlama 10.16%, CodeQwen1.5 4.82%, DeepSeek Coder 30.76%, DeepSeek Chat 37.93% — all below 40%, meaning >60% generate code (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "Pass@1 of most Code LLMs drops by 35%–52% and Test Pass Rate drops by 17%–35% on HumanEvalComm compared to original HumanEval, with statistical significance.",
       "evidence": "Tables 3-5 show relative drops with p-values. E.g., ChatGPT Pass@1: 65.58% → 31.34%; CodeQwen1.5: 76.83% → 47.61%. Statistical significance at p<0.01 for most comparisons (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "Okanagan (ChatGPT base) increases Communication Rate by 58% and Good Question Rate by 38% absolute over ChatGPT alone.",
       "evidence": "Table 3: Okanagan Communication Rate 72.73% vs ChatGPT 14.21% (diff ≈58%). Good Question Rate 52.24% vs 13.43% (diff ≈39%). Numbers match abstract claims.",
       "supported": "strong"
     },
     {
       "claim": "Okanagan boosts Pass@1 by 8% and Test Pass Rate by 7% absolute over ChatGPT on HumanEvalComm.",
       "evidence": "Table 3: Okanagan Pass@1 39.62% vs ChatGPT 31.34% (diff ≈8%). Test Pass Rate 56.98% vs 49.39% (diff ≈8%). However, LLM-based evaluator has 0.94% False Recovery Rate for ChatGPT but 30-40% for other models (Table 6), introducing potential confound.",
       "supported": "moderate"
     },
     {
       "claim": "Incompleteness category results in higher communication rates but lower pass rates than Ambiguity and Inconsistency categories.",
       "evidence": "Table 4: For most models, 1p (Incompleteness) has higher communication rates (e.g., ChatGPT: 31.68% for 1p vs 5.84% for 1a vs 5.84% for 1c) and lower Pass@1 (e.g., ChatGPT: 27.95% for 1p vs 33.77% for 1a vs 53.25% for 1c).",
       "supported": "strong"
     },
     {
       "claim": "LLM-based evaluator's Good Question Rate aligns with human judgment within 10% for Communication Rate but shows larger discrepancies for Good Question Rate.",
       "evidence": "Figure 5 shows Communication Rate differences <10% between automated and manual evaluation. Good Question Rate shows larger discrepancies for CodeLlama, CodeQwen1.5, DeepSeek Coder (Section 4.3).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Code LLMs overwhelmingly generate code (>60%) rather than ask clarifying questions even when problem descriptions are deliberately made ambiguous, inconsistent, or incomplete, with Pass@1 dropping 35-52%. The proposed LLM agent approach Okanagan increases communication rate from 14% to 73% and modestly improves code generation metrics (+8% Pass@1) by asking clarifying questions before coding. However, Okanagan also asks unnecessary questions on well-specified problems, and the LLM-based evaluator used for assessment shows non-trivial biases (30-40% False Recovery Rate for some models).",
   "red_flags": [
     {
       "flag": "Internal number inconsistency",
       "detail": "The abstract reports Okanagan increases Good Question Rate by 38%, but the body text in Section 1 states '5%' for the same metric. This is a significant discrepancy that is never reconciled."
     },
     {
       "flag": "Circular evaluation with GPT-3.5",
       "detail": "GPT-3.5 serves triple duty: it is the base model for ChatGPT experiments, the base model for Okanagan, and the LLM-based evaluator that rates question quality and generates answers. This creates a potential self-evaluation bias where the evaluator may be more generous toward outputs from its own architecture."
     },
     {
       "flag": "High False Recovery Rate inflates some baselines",
       "detail": "Table 6 shows 30-40% False Recovery Rate for CodeLlama, CodeQwen1.5, DeepSeek Coder, and DeepSeek Chat — the evaluator provides correct answers even when these models asked no relevant questions, artificially inflating their pass rates relative to Okanagan (0% False Recovery Rate)."
     },
     {
       "flag": "No contamination analysis despite HumanEval-based benchmark",
       "detail": "HumanEvalComm is built on HumanEval (published 2021). All evaluated models were trained after 2021 and likely saw original HumanEval solutions. The modifications reduce but do not eliminate contamination risk, and this is never discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark used as the basis for HumanEvalComm; foundational code generation evaluation work."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "Major real-world code generation benchmark; cited as example of benchmarks that don't evaluate communication skills."
     },
     {
       "title": "SWE-Agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Prominent LLM agent for software engineering tasks, relevant to agentic code generation evaluation."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the evaluated Code LLMs; key open-source code generation model."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the evaluated Code LLMs; achieved top communication rates among base models."
     },
     {
       "title": "AgentCoder: Multi-agent-based code generation with iterative testing and optimisation",
       "authors": ["Dong Huang", "Qingwen Bu"],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "LLM agent baseline for code generation that was tested on HumanEvalComm but showed 0% communication rate."
     },
     {
       "title": "LLM is Like a Box of Chocolates: the Non-determinism of ChatGPT in Code Generation",
       "authors": ["Shuyin Ouyang", "Jie M Zhang"],
       "year": 2023,
       "arxiv_id": "2308.02828",
       "relevance": "Study of ChatGPT non-determinism in code generation; provided code used in this paper's evaluation."
     },
     {
       "title": "Beyond accuracy: Evaluating self-consistency of code large language models with identitychain",
       "authors": ["Marcus J Min", "Yangruibo Ding"],
       "year": 2023,
       "arxiv_id": "2310.14053",
       "relevance": "Evaluates self-consistency of Code LLMs beyond accuracy; provided code used in this paper's evaluation."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Nelson Labash"],
       "year": 2023,
       "arxiv_id": "2303.11366",
       "relevance": "Framework for LLM self-improvement through verbal reinforcement; related agent approach for code generation."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao"],
       "year": 2023,
       "arxiv_id": "2210.03629",
       "relevance": "Foundational work on integrating reasoning and acting in LLMs; underpins many agentic code generation approaches."
     },
     {
       "title": "LLM-based Test-driven Interactive Code Generation: User Study and Empirical Evaluation",
       "authors": ["Sarah Fakhoury", "Aaditya Naik"],
       "year": 2024,
       "arxiv_id": "2404.10100",
       "relevance": "Proposes interactive test-driven code generation with LLMs, closely related to communication-enhanced code generation."
     },
     {
       "title": "Rethinking Software Engineering in the Foundation Model Era: From Task-Driven AI Copilots to Goal-Driven AI Pair Programmers",
       "authors": ["Ahmed E Hassan", "Gustavo A Oliva"],
       "year": 2024,
       "arxiv_id": "2404.10225",
       "relevance": "Discusses the shift from task-driven to goal-driven AI programming assistants; frames the communication capability gap this paper addresses."
     }
   ]
 }