ai-research-survey

scan-v5.json (22442B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "HumanEvalComm: Benchmarking the Communication Competence of Code Generation for LLMs and LLM Agents",
     "authors": [
       "J. Wu",
       "Fatemeh H. Fard"
     ],
     "year": 2024,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2406.00215",
     "doi": "10.1145/3715109"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All quantitative claims in the abstract (>60% generate code, 35-52% Pass@1 drop, absolute 58%/38% improvement for Okanagan) are directly traceable to Tables 3-5 with statistical significance reported.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Okanagan 'boosts' pass rates by asking clarifying questions, but the False Recovery Rate problem (30-40% for some models) means the evaluator sometimes provides missing info even without questions being asked, confounding the causal mechanism.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is built exclusively on HumanEval (164 Python problems, algorithmic tasks) yet the title and framing claim general communication competence evaluation; the external validity section acknowledges non-generalizability only briefly.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for anomalous results (DeepSeek Chat improvement attributed to formatting failures in baseline, LLM evaluator inflation affecting Good Question Rate) and acknowledges the generative nature of LLMs as explaining low communication rates.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper distinguishes between Communication Rate (proxy for asking behavior), Good Question Rate (proxy for question quality), and Pass@1/Test Pass Rate (functional correctness), explicitly discussing how each measures a different aspect and where they diverge from ground truth.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 'Limitations' and Section 7 'Threats to Validity' both exist as dedicated sections addressing separate concerns about the benchmark and evaluation.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named: manual modification subjectivity, parameter choices for open-source models (max_new_tokens=512), LLM evaluator tendency to over-label 'Good' questions, and False Recovery Rate inflating pass rates for specific models (CodeQwen1.5, DeepSeek Coder).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states results cannot be claimed for other datasets without further testing, and the benchmark is restricted to Python/HumanEval-style algorithmic problems with three specific clarification types.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in acknowledgments: 'Natural Sciences and Engineering Research Council of Canada RGPIN-2019-05175.'",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors list University of British Columbia, Kelowna affiliation; no evaluated product is affiliated with the authors.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSERC is a Canadian government research council with no stake in the benchmark results or the models evaluated.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests declaration is present; the acknowledgment only mentions the grant and student helpers.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Communication capability is formally defined as how well a model asks clarifying questions when requirements are incomplete/inconsistent/ambiguous; Communication Rate and Good Question Rate are precisely defined with formulas; the three clarification types have explicit definitions drawn from Requirements Engineering literature.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are enumerated: (1) HumanEvalComm benchmark, (2) Okanagan LLM agent approach, (3) first empirical study on communication competence with LLM-based evaluator and new metrics.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 covers code generation LLMs, self-correction, and LLM agents for code generation, situating the work as the 'first systematic empirical study' on communication skills and distinguishing it from prior task-solving agent work.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that communication competence in software engineering requires asking clarifying questions under ambiguous/incomplete/inconsistent requirements, grounded in Requirements Engineering literature, and the benchmark is designed to operationalize this by creating problems that require clarification to solve correctly.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "Difficulty is implicitly ordered (2-type combinations harder than 1-type, Incompleteness harder than Inconsistency for pass rates) but no formal difficulty characterization, tiers, or difficulty metrics are reported; easy/medium/hard is not quantified.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "CodeQwen1.5 Chat achieves 0% Communication Rate for 1p (floor effect in communication) and some models approach very low pass rates; the paper notes results but does not explicitly check for or discuss ceiling/floor effects as a benchmark design concern.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline for the main communication task is included; the 6 graduate students are used only to validate the LLM-based evaluator, not as a performance baseline for asking clarifying questions.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Communication Rate (code vs. non-code binary) and Good Question Rate (Good/Fair/Bad via LLM evaluator) are defined and their limitations discussed in detail in RQ3, including comparison with human judgment to validate the rubric.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "No contamination resistance is designed in; the benchmark is derived from HumanEval which is widely used in training, and the paper does not analyze whether models have seen original HumanEval problems during pre-training.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Temporal robustness is not discussed; the paper does not address how the benchmark will remain useful as models improve or become capable of consistently asking clarifying questions.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.3 and 5.5 discuss key failure modes: LLM evaluator over-labeling Good questions, False Recovery Rate inflating pass rates for non-communicating models, and Okanagan asking unnecessary questions for clear problems.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full replication package including benchmark data, evaluation code, and Okanagan implementation is publicly released at https://github.com/jie-jw-wu/human-eval-comm.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": false,
           "justification": "The construction methodology is described (Section 2, ~100 hours manual modification, 2 annotators), statistics are given (Table 1), but no formal data card, preprocessing steps, or storage format documentation is provided.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark is stated to be publicly available on GitHub but no specific license (e.g., MIT, Apache, CC-BY) is mentioned in the paper, creating ambiguity about permitted use.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.5 notes other researchers can use HumanEvalComm directly without replication effort; Section 6 specifies it should evaluate communication competence in code generation; limitations on generalizability to other datasets are stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "More than 60% of Code LLM responses generate code rather than ask clarifying questions when problem descriptions have clarification issues.",
       "evidence": "Table 3 shows Communication Rates of 14.21% (ChatGPT), 10.16% (CodeLlama), 4.82% (CodeQwen1.5), 30.76% (DeepSeek Coder), 37.93% (DeepSeek Chat) — all well below 50%, meaning >62-95% generate code instead.",
       "supported": "strong"
     },
     {
       "claim": "Pass@1 and Test Pass Rate of Code LLMs drop by 35-52% and 17-35% respectively on modified problems.",
       "evidence": "Table 3 shows, e.g., ChatGPT Pass@1: 65.58% → 31.34% (52% relative drop); CodeQwen1.5 Chat: 76.83% → 47.61% (38% drop). Statistical significance reported for over 75% of comparisons.",
       "supported": "strong"
     },
     {
       "claim": "Okanagan increases Communication Rate and Good Question Rate by absolute 58% and 38% compared to ChatGPT 3.5.",
       "evidence": "Table 3: ChatGPT Communication Rate 14.21% vs Okanagan 72.73% (+58.52%); Good Question Rate 13.43% vs 52.24% (+38.81%).",
       "supported": "strong"
     },
     {
       "claim": "Okanagan boosts Pass@1 and Test Pass Rate by absolute 8% and 7% over ChatGPT on modified problems.",
       "evidence": "Table 3: Okanagan Pass@1 39.62% vs ChatGPT 31.34% (+8.28%); Test Pass Rate 56.98% vs 49.39% (+7.59%).",
       "supported": "strong"
     },
     {
       "claim": "Incompleteness triggers higher communication rates but results in lower pass rates than Ambiguity and Inconsistency.",
       "evidence": "Table 4 shows for ChatGPT: Communication Rate 1p=31.68% vs 1a=5.84% vs 1c=5.84%; but Test Pass Rate 1p=44.14% < 1a=54.98% < 1c=66.37%.",
       "supported": "strong"
     },
     {
       "claim": "Even with 90% of problem description randomly removed, 46% of LLM responses still generate code without asking questions.",
       "evidence": "Table 7 shows Communication Rate at 90% removal = 54.1%, meaning 45.9% still generate code; this is using ChatGPT 3.5.",
       "supported": "moderate"
     },
     {
       "claim": "The LLM-based evaluator has high False Recovery Rates (30-40%) for some models, potentially inflating pass rates.",
       "evidence": "Table 6 shows False Recovery Rate for CodeLlama=32.5%, CodeQwen1.5=40.7%, DeepSeek Coder=38.0%, DeepSeek Chat=34.8%, while Okanagan=0%.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "Current Code LLMs have severely limited communication capabilities: over 60% of responses generate code even when problem descriptions are demonstrably incomplete, inconsistent, or ambiguous, and performance drops 35-52% on Pass@1. The Okanagan LLM agent (multi-round with clarification step) increases communication rates by ~58 percentage points and improves pass rates, but introduces a new failure mode of asking unnecessary questions on clear problems. The LLM-based evaluator approach is reasonable but exhibits systematic biases including over-labeling questions as 'Good' and high False Recovery Rates for non-communicating models, complicating metric reliability.",
   "red_flags": [
     {
       "flag": "LLM evaluator False Recovery Rate",
       "detail": "The LLM-based evaluator provides answers that recover missing information even when models ask no questions, with False Recovery Rates of 32-41% for CodeLlama, CodeQwen1.5, and DeepSeek Coder. This artificially inflates Pass@1 and Test Pass Rate for non-communicating models, making comparisons unreliable."
     },
     {
       "flag": "No contamination analysis",
       "detail": "HumanEvalComm is derived from HumanEval, one of the most widely used code generation benchmarks in LLM training corpora. The paper does not analyze whether models have memorized HumanEval problems, which could affect both baseline and modified problem performance."
     },
     {
       "flag": "No human baseline for main task",
       "detail": "There is no human performance baseline on HumanEvalComm itself — human participants are only used to validate the evaluator quality, not to anchor what good performance looks like on the communication task."
     },
     {
       "flag": "Single-domain benchmark overgeneralization",
       "detail": "All 762 modified problems are Python algorithmic tasks from HumanEval; the paper title and framing imply broad 'communication competence' evaluation, but the external validity section acknowledges results may not generalize to other programming languages, problem types, or real-world software engineering tasks."
     },
     {
       "flag": "Okanagan only tested with ChatGPT 3.5",
       "detail": "The primary Okanagan results use ChatGPT 3.5 as the base LLM; the DeepSeek Coder variant underperforms, and the paper attributes this to prompt sensitivity without fully investigating it, limiting generalizability of the agent approach."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code (HumanEval)",
       "relevance": "Foundation benchmark that HumanEvalComm is built upon; defines Pass@k metric used throughout"
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "relevance": "Related code generation benchmark representing real-world software engineering tasks; used as contrast to show current benchmarks don't evaluate communication"
     },
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "relevance": "Survey of LLM applications in SE identifying communication gaps as open problem"
     },
     {
       "title": "AgentCoder: Multi-agent-based code generation with iterative testing and optimisation",
       "relevance": "LLM agent baseline evaluated on HumanEvalComm; shows 0% communication rate when adapted to the task"
     },
     {
       "title": "SWE-Agent: Agent-computer interfaces enable automated software engineering",
       "relevance": "Related LLM agent for software engineering used as context for agent-based approaches"
     },
     {
       "title": "Exploring collaboration mechanisms for LLM agents: A social psychology view",
       "relevance": "Framework that inspires Okanagan's multi-round debate/reflection structure"
     },
     {
       "title": "Beyond accuracy: Evaluating self-consistency of code large language models with IdentityChain",
       "relevance": "Provides evaluation methodology reused for open-source model testing in the study"
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "relevance": "Survey of LLM agent types and capabilities, providing background for the agent evaluation framework"
     },
     {
       "title": "LLM is Like a Box of Chocolates: the Non-determinism of ChatGPT in Code Generation",
       "relevance": "Provides metrics (Mean, Variance, Max Diff, Ratio of Worst) reused for the incomplete modification experiments in RQ4"
     },
     {
       "title": "Rethinking Software Engineering in the Foundation Model Era",
       "relevance": "Argues for goal-driven AI pair programmers vs task-driven copilots, directly motivating the communication competence framing"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners using LLM coding assistants can directly apply the finding that models almost never ask clarifying questions, and Okanagan is available on GitHub as a drop-in improvement."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that LLMs continue generating code even with 90% of the description removed challenges assumptions about LLM comprehension capabilities and prompts a reassessment of how 'capable' current models actually are."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or AI risk concerns raised; the paper focuses on software quality rather than harmful behaviors."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild tension between the narrative of LLMs as capable coders vs. the evidence they can't ask basic clarifying questions, but no direct controversy with other researchers."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Full benchmark and Okanagan code are publicly available on GitHub; anyone can run their own model against HumanEvalComm immediately."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "University of British Columbia is a respected institution but not a famous AI lab; no big-tech affiliation or celebrity researchers."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45291024",
         "title": "Launch HN: Cactus (YC S25) – AI inference on smartphones",
         "points": 123,
         "comments": 63,
         "url": "https://news.ycombinator.com/item?id=45291024",
         "created_at": "2025-09-18T15:40:29Z"
       },
       {
         "hn_id": "44430311",
         "title": "Small language models are the future of agentic AI",
         "points": 113,
         "comments": 45,
         "url": "https://news.ycombinator.com/item?id=44430311",
         "created_at": "2025-07-01T03:33:49Z"
       },
       {
         "hn_id": "36165862",
         "title": "The feasibility of artificial consciousness through the lens of neuroscience",
         "points": 5,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=36165862",
         "created_at": "2023-06-02T14:49:24Z"
       },
       {
         "hn_id": "44246361",
         "title": "Small Language Models Are the Future of Agentic AI",
         "points": 5,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44246361",
         "created_at": "2025-06-11T11:16:33Z"
       },
       {
         "hn_id": "41137040",
         "title": "Positive Mass in General Relativity Without Energy Conditions",
         "points": 3,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=41137040",
         "created_at": "2024-08-02T08:19:18Z"
       },
       {
         "hn_id": "39941576",
         "title": "Jailbreaking Leading Safety-Aligned LLMs with Simple Adaptive Attacks",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=39941576",
         "created_at": "2024-04-05T12:30:05Z"
       },
       {
         "hn_id": "40310614",
         "title": "The AI Review Lottery: Widespread AI-Assisted Peer Reviews Boost Paper Scores",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40310614",
         "created_at": "2024-05-09T17:31:58Z"
       },
       {
         "hn_id": "45549900",
         "title": "Agentic web browsing can't scale with cloud LLMs",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45549900",
         "created_at": "2025-10-11T15:29:17Z"
       },
       {
         "hn_id": "36179769",
         "title": "The feasibility of artificial consciousness through the lens of neuroscience",
         "points": 1,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=36179769",
         "created_at": "2023-06-03T19:30:29Z"
       },
       {
         "hn_id": "23527095",
         "title": "MLOS: An Infrastructure for Automated Software Performance Engineering",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=23527095",
         "created_at": "2020-06-15T13:31:04Z"
       }
     ],
     "top_points": 123,
     "total_points": 257,
     "total_comments": 116
   }
 }