ai-research-survey

scan.json (27123B)

---

 {
   "paper": {
     "title": "ClarifyGPT: Empowering LLM-based Code Generation with Intention Clarification",
     "authors": [
       "Fangwen Mu",
       "Lin Shi",
       "Song Wang",
       "Zhuohao Yu",
       "Binquan Zhang",
       "ChenXue Wang",
       "Shichao Liu",
       "Qing Wang"
     ],
     "year": 2023,
     "venue": "arXiv preprint (submitted to ACM)",
     "arxiv_id": "2310.10996",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided: https://github.com/ClarifyGPT/ClarifyGPT (Reference [1]). The paper states 'publicly accessible dataset and source code [1] to facilitate the replication of our study.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (HumanEval, MBPP-sanitized, HumanEval-ET, MBPP-ET) and the GitHub repository is stated to contain the dataset. The benchmarks themselves are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model configuration parameters (temperature, top_p, max_tokens) but does not provide a requirements.txt, Dockerfile, or detailed environment setup section listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper references the GitHub repository but does not include a 'Reproducing Results' section or specific commands to run experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., 80.80%, 60.19%) across Tables 2, 3, and 4 with no confidence intervals, error bars, or ± notation despite running each approach three times."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ClarifyGPT 'significantly' improves performance and 'substantially' improves results but provides no statistical significance tests (no p-values, t-tests, or other tests). Comparisons are based solely on comparing percentage point differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baseline context throughout. For example: 'ClarifyGPT elevates the performance (Pass@1) of GPT-4 from 70.96% to 80.80%' (Table 2), with relative improvement percentages (13.87%, 16.83%). This provides enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation uses only 10 participants with no justification for why this number was chosen and no power analysis. The number of benchmark problems is inherited from existing benchmarks without justification for adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we run each approach three times and report the average results as the final results' (Section 4.4) but does not report standard deviation, variance, or any spread measure across the three runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are included: Default LLM, Chain-of-Thought (CoT), and GPT-Engineer (Section 4.5). Comparisons are shown in Tables 2 and 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are contemporary: CoT (Wei et al. 2022) and GPT-Engineer (Osika, 2023) are recent approaches. GPT-Engineer is the most directly comparable system for interactive code generation."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed to measure the individual contribution of the main components (test input generation, code consistency check, reasoning-based question generation). RQ3 varies the number of demonstrations but does not ablate framework components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Pass@1 is used as the evaluation metric. No other metrics (e.g., Pass@k for k>1, code quality metrics, question quality metrics) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 (Section 5.1) includes a human evaluation with 10 participants who use ClarifyGPT for code generation on MBPP-sanitized and MBPP-ET benchmarks. Participants answered clarifying questions generated by the system."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks use dedicated test cases for evaluation. The three demonstration examples are taken from the first three problems of each benchmark (Section 4.6), while the remaining problems serve as the evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (HumanEval, HumanEval-ET, MBPP-sanitized, MBPP-ET) and per model (ChatGPT, GPT-4) in Tables 2, 3, and 4, rather than only reporting overall averages."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 (Case Study) discusses qualitative examples including cases where baselines fail. Section 6.2 (Limitations) discusses cases where ClarifyGPT is not suitable, such as code with complex input types or code that does not return output values."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that ClarifyGPT in the zero-shot setting shows only marginal improvement (e.g., 0.5% on HumanEval with GPT-4, 0.0% on HumanEval-ET), effectively near-zero improvement without demonstrations (Table 4). The paper also notes simulated feedback performs slightly worse than human feedback."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GPT-4 performance improves from 70.96% to 80.80% on MBPP-sanitized (supported by Table 2), and average improvement from 68.02% to 75.75% for GPT-4 and 58.55% to 67.22% for ChatGPT (supported by Table 3). All numerical claims match the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'ClarifyGPT can effectively facilitate the practical application of LLMs' and attributes improvements to specific components ('We attribute the improvements to our novel techniques, i.e., ambiguous requirement identification and clarifying question generation'). However, without an ablation study, the causal contribution of individual components is not established."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'ClarifyGPT can significantly facilitate the practical application of LLMs in real-world development environments' but tests only on Python function-level code generation benchmarks with only two LLMs (GPT-4 and ChatGPT). The title 'Empowering LLM-based Code Generation' is broader than what is tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (Section 6.3) discusses data leakage, simulation fidelity, and generalizability but does not consider alternative explanations for the observed improvements. For instance, the improvement could partly stem from additional LLM calls (more compute) rather than the specific clarification mechanism. No such alternatives are discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'gpt-3.5-turbo' and 'gpt-4-turbo' (Section 4.2) but does not specify snapshot dates or exact version identifiers (e.g., gpt-3.5-turbo-0613 or gpt-4-turbo-2023-xx-xx). These API model names can change behavior over time."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows the detailed prompts used in ClarifyGPT for all four stages: seed input initialization, reasoning-based question generation, user simulation, and enhanced code generation. The actual prompt text with instructions, demonstrations, and query structure are provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 reports: top_p=0.95, frequency_penalty=0, max_tokens=800 for question generation and 300 for others, temperature=0 for most operations and 0.8 for code sampling. Stop sequences are also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The four-stage pipeline (test input generation, code consistency check, reasoning-based question generation, enhanced code generation) is described in detail in Section 3 with Figure 1 providing an overview. The workflow, prompting strategy, and mutation logic are all documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 describes prompt construction: demonstration seeds are selected from the first three problems of each benchmark, and demonstrations are manually created. Section 4.3 describes the benchmarks and their characteristics. The data pipeline from requirement input to code output is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 'Benefits and Limitations' provides substantive discussion of limitations, and Section 6.3 'Threats to Validity' discusses three specific threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses specific threats: (1) data leakage from LLM training data overlapping with benchmarks, (2) user simulation fidelity, and (3) generalizability across only two LLMs and four datasets. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly states that ClarifyGPT requires instruction-tuned LLMs (not applicable to InCoder, CodeGen), is not suitable for code with complex inputs (images, files), and may have limitations for code without return values (e.g., deep learning programs)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmarks are public, the raw experimental outputs (generated code solutions, generated questions, participant responses from the human evaluation) are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the human evaluation data collection in detail: 140 ambiguous problems identified, 2.85 average questions per problem, three questionnaires per problem, and participant assignment of 42 problems each."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the 10 participants: three PhD students, two Master's students, two senior researchers, and three industry developers. All have at least 3 years of Python experience (6 with 5+ years). They were trained with example questionnaires before the study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from requirement input through ambiguity detection, question generation, participant answers, to final code generation is documented. Section 5.1 describes the flow: 427 problems → 140 identified as ambiguous → questionnaires distributed → answers collected → code generated → evaluated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper. One co-author (Shichao Liu) is from Huawei, suggesting potential industry funding, but no funding source is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed on page 1, including Shichao Liu from 'Software IDE innovation Lab, Huawei Central Software Institute, China.' Academic affiliations at CAS and Beihang University are also listed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "One author is from Huawei, a company that develops and sells AI/LLM-powered development tools. No funding disclosure is provided, so independence of funding from outcomes cannot be assessed. The paper evaluates OpenAI models rather than Huawei products, but the general area of LLM-based code generation is commercially relevant to Huawei."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for GPT-4 or ChatGPT (gpt-3.5-turbo). This is relevant because the benchmarks (HumanEval, MBPP) were published before GPT-4's training cutoff."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses this: 'Since these LLMs are trained on open-source code repositories, it is possible that some public benchmarks were included in their training data.' The paper acknowledges the potential for overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While Section 6.3 acknowledges the contamination risk, the mitigation offered ('we carefully select HumanEval and MBPP-sanitized') is insufficient. HumanEval was published in 2021 and MBPP in 2021 — both were very likely in GPT-4's training data. No actual contamination analysis (e.g., canary strings, membership inference) is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. The human evaluation with 10 participants is not pre-registered on any platform (OSF, AsPredicted, etc.)."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite the study involving 10 human participants who answered questionnaires."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports demographics: three PhD students, two Master's students, two senior researchers, three industry developers. Experience levels are noted: all have 3+ years of Python experience, six with 5+ years."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "The paper states participants have 'at least three years of experience in Python development' but does not describe formal inclusion/exclusion criteria or a screening process. The 10 participants were 'recruited' without description of how they were selected."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '42 problems' were assigned to each participant and 'Each problem will be solved by three participants' but does not describe how the assignment was performed or whether it was randomized."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Participants knew they were answering clarifying questions generated by ClarifyGPT. Whether they knew which system produced which output is not discussed."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information is reported. The paper does not state whether all 10 participants completed all their assigned tasks or if any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency measurements are reported despite ClarifyGPT requiring multiple LLM calls per problem (seed input generation, code sampling, question generation, user simulation, final code generation). The paper acknowledges GPT-Engineer 'results in an increase in token counts...consequently escalating operational expenses' but does not report ClarifyGPT's own costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware information is stated. The experiments involve multiple LLM API calls across four benchmarks, two models, three runs each, but the total cost is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "ClarifyGPT elevates GPT-4 Pass@1 on MBPP-sanitized from 70.96% to 80.80% with human feedback.",
       "evidence": "Table 2 (Section 5.1) shows the human evaluation results across 10 participants on MBPP-sanitized benchmark.",
       "supported": "moderate"
     },
     {
       "claim": "ClarifyGPT improves the average performance of GPT-4 across four benchmarks from 68.02% to 75.75% with simulated feedback.",
       "evidence": "Table 3 (Section 5.2) shows automated evaluation results: HumanEval 87.80%, HumanEval-ET 78.05%, MBPP-sanitized 78.69%, MBPP-ET 58.47%, averaging 75.75%.",
       "supported": "moderate"
     },
     {
       "claim": "ClarifyGPT improves the average performance of ChatGPT across four benchmarks from 58.55% to 67.22% with simulated feedback.",
       "evidence": "Table 3 (Section 5.2) shows: HumanEval 74.39%, HumanEval-ET 64.84%, MBPP-sanitized 74.08%, MBPP-ET 55.58%, averaging 67.22%.",
       "supported": "moderate"
     },
     {
       "claim": "ClarifyGPT's user simulation method produces high-fidelity results close to human feedback.",
       "evidence": "Comparison between ClarifyGPT (Human Feedback) and ClarifyGPT (Simulated Feedback) in Table 3 shows simulated results are slightly lower but in a similar range (e.g., 78.69% vs 80.80% on MBPP-sanitized with GPT-4).",
       "supported": "moderate"
     },
     {
       "claim": "ClarifyGPT demonstrates robustness to the number of demonstrations in prompts.",
       "evidence": "Table 4 (Section 5.3) shows consistent improvements from zero-shot to three-shot across all benchmarks and models.",
       "supported": "moderate"
     },
     {
       "claim": "ClarifyGPT can significantly facilitate the practical application of LLMs in real-world development environments.",
       "evidence": "Only tested on Python function-level benchmark problems (HumanEval, MBPP) with two LLMs. No real-world development environment testing was conducted.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "ClarifyGPT is a framework that enhances LLM-based code generation by detecting ambiguous requirements via code consistency checking and generating targeted clarifying questions. In human evaluation with 10 participants, ClarifyGPT improved GPT-4 Pass@1 on MBPP-sanitized from 70.96% to 80.80%. Automated evaluations using a user simulation method showed average improvements of 11.52% for GPT-4 and 15.07% for ChatGPT across four Python code generation benchmarks. The framework's performance scales with the number of few-shot demonstrations, with one-shot already capturing most of the improvement.",
   "red_flags": [
     {
       "flag": "No variance or error bars despite multiple runs",
       "detail": "The paper runs each experiment three times and reports averages but never reports standard deviation, confidence intervals, or any spread measure. Without this, the reader cannot assess whether observed differences are reliable or within noise."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "The paper uses words like 'significantly' and 'substantially' to describe improvements but provides no statistical tests. Differences between methods could be within random variation."
     },
     {
       "flag": "No ablation study",
       "detail": "The framework has four distinct stages but no ablation study isolates the contribution of each component. It is unclear whether the improvement comes from the consistency check, the question generation, the reasoning-based prompting, or simply from additional LLM calls and refined prompts."
     },
     {
       "flag": "User simulation provides ground truth information",
       "detail": "The simulated user feedback method gives the LLM access to ground-truth test cases when generating responses. This creates an unfair advantage: the simulated user has perfect knowledge of the expected behavior, which real users would not have. The simulated feedback results may overestimate real-world performance."
     },
     {
       "flag": "Small human evaluation sample",
       "detail": "Only 10 participants were used for the human evaluation, with no power analysis or justification for this number. The study has no IRB approval, no formal inclusion/exclusion criteria, and no attrition reporting."
     },
     {
       "flag": "Potential benchmark contamination unaddressed",
       "detail": "HumanEval (2021) and MBPP (2021) were very likely in the training data of GPT-4 and ChatGPT. The paper acknowledges this risk but performs no contamination analysis. Improvements may partly reflect the model already knowing the answers and benefiting from disambiguating hints."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational work for evaluating LLM code generation capabilities."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used for evaluating code generation."
     },
     {
       "title": "Chain of Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting technique (CoT) used as a baseline and inspiration for ClarifyGPT's reasoning-based prompting."
     },
     {
       "title": "CodeT: Code Generation with Generated Tests",
       "authors": ["Bei Chen", "Fengji Zhang"],
       "year": 2022,
       "arxiv_id": "2207.10397",
       "relevance": "Related approach using generated tests to improve code generation via dual execution agreement."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Introduces extended test case benchmarks (HumanEval-ET, MBPP-ET) and type-aware mutation strategy used by ClarifyGPT."
     },
     {
       "title": "Self-collaboration Code Generation via ChatGPT",
       "authors": ["Yihong Dong", "Xue Jiang"],
       "year": 2023,
       "arxiv_id": "2304.07590",
       "relevance": "Related work on enhancing LLM code generation through multi-agent collaboration."
     },
     {
       "title": "Python Code Generation by Asking Clarification Questions",
       "authors": ["Haau-Sing Li", "Mohsen Mesgar", "André F. T. Martins", "Iryna Gurevych"],
       "year": 2023,
       "doi": "10.18653/v1/2023.acl-long.799",
       "relevance": "The only prior work addressing ambiguous requirements in code generation; directly compared and contrasted with ClarifyGPT."
     },
     {
       "title": "CODAMOSA: Escaping coverage plateaus in test generation with pre-trained large language models",
       "authors": ["Caroline Lemieux", "Jeevana Priya Inala", "Shuvendu K. Lahiri", "Siddhartha Sen"],
       "year": 2023,
       "relevance": "LLM-based test generation approach relevant to understanding automated testing with language models."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, one of the two primary LLMs evaluated in this study."
     },
     {
       "title": "Interactive Code Generation via Test-Driven User-Intent Formalization",
       "authors": ["Shuvendu K. Lahiri", "Aaditya Naik"],
       "year": 2022,
       "arxiv_id": "2208.05950",
       "relevance": "Related interactive code generation approach using test-driven formalization of user intent."
     },
     {
       "title": "Enabling Programming Thinking in Large Language Models Toward Code Generation",
       "authors": ["Jia Li", "Ge Li"],
       "year": 2023,
       "relevance": "Structured Chain-of-Thought (SCoT) for code generation, a prompting technique compared conceptually with ClarifyGPT's approach."
     },
     {
       "title": "Competition-Level Code Generation with AlphaCode",
       "authors": ["Yujia Li", "David H. Choi"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Large-scale code generation system demonstrating sampling and filtering approaches to code generation."
     }
   ]
 }