ai-research-survey

scan.json (19950B)

---

 {
   "paper": {
     "title": "ClarifyGPT: Empowering LLM-based Code Generation with Intention Clarification",
     "authors": ["Fangwen Mu", "Lin Shi", "Song Wang", "Zhuohao Yu", "Binquan Zhang", "ChenXue Wang", "Shichao Liu", "Qing Wang"],
     "year": 2023,
     "venue": "arXiv preprint",
     "arxiv_id": "2310.10996"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/ClarifyGPT/ClarifyGPT (reference [1])."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (HumanEval, MBPP-sanitized, HumanEval-ET, MBPP-ET) and states 'publicly accessible dataset and source code' in contributions."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section found. Only API model names are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions found in the paper. The GitHub link is provided but the paper itself lacks a reproducing results section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 2-4 report only point estimates (e.g., 80.80%) with no confidence intervals or error bars, despite running experiments three times."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ClarifyGPT outperforms baselines but provides no statistical significance tests. Comparisons are based solely on point estimate differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative improvements are reported with baseline context, e.g., 'improves the average performance of GPT-4 across four benchmarks from 68.02% to 75.75%' (Section 5.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 10 participants were recruited for the human evaluation, nor why 3 runs were chosen for averaging."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we run each approach three times and report the average results' (Section 4.4) but does not report standard deviation or any spread measure across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines compared: Default LLM, Chain-of-Thought (CoT), and GPT-Engineer (Section 4.5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CoT (2022) and GPT-Engineer (2023) are contemporary baselines. GPT-Engineer is the most directly related work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 varies the number of demonstrations (zero to three-shot), functioning as an ablation of the prompt design component (Table 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only Pass@1 is used as the evaluation metric. No other metrics (e.g., Pass@k for k>1, code quality measures) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 involves 10 human participants answering clarifying questions and evaluating code generation on MBPP-sanitized and MBPP-ET (Section 5.1)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses standard benchmark test suites (HumanEval, MBPP-sanitized, etc.) which are separate from the demonstration examples selected from the first three problems."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (HumanEval, HumanEval-ET, MBPP-sanitized, MBPP-ET) in Tables 2-4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 case study shows both success and failure cases. Section 6.2 Limitations discusses where ClarifyGPT fails (complex inputs, code without return values)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The zero-shot setting shows marginal improvement (0.0-2.8%), reported honestly in Table 4. The paper also notes ClarifyGPT (Simulated Feedback) underperforms Human Feedback."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (GPT-4 from 70.96% to 80.80% on MBPP-sanitized; average improvements of 11.52% and 15.07%) match Tables 2 and 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('ClarifyGPT improves code generation') are supported by controlled comparisons where ClarifyGPT is the only variable changed against Default, CoT, and GPT-Engineer baselines."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims ClarifyGPT 'can effectively facilitate the practical application of LLMs in real-world development environments' but tests only on Python function-level benchmarks with short requirements. Section 6.2 acknowledges limitations but the abstract/conclusion overgeneralize."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (6.3) discusses data leakage and simulation fidelity but does not consider alternative explanations for the observed improvements (e.g., additional tokens/context providing more information regardless of clarification quality)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 specifies 'gpt-3.5-turbo' for ChatGPT and 'gpt-4-turbo' for GPT-4, though no snapshot dates are given. These are API model names, not versioned snapshots."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows the actual prompt templates used for seed input initialization, question generation, user simulation, and enhanced code generation, including instruction text and demonstration structure."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 reports top_p=0.95, frequency_penalty=0, max_tokens=800/300, temperature=0 (or 0.8 for sampling)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The four-stage pipeline (test input generation, code consistency check, reasoning-based question generation, enhanced code generation) is described in detail in Section 3 with Figure 1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 describes prompt construction (selecting first three problems as demonstration seeds). Section 4.3 describes benchmark statistics. The process from benchmark to evaluation is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 'Benefits and Limitations' and Section 6.3 'Threats to Validity' provide substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses specific threats: data leakage from training on public benchmarks, simulation fidelity concerns, and generalizability across only two LLMs and four datasets."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 Limitations explicitly states ClarifyGPT is not suitable for code with complex inputs (images, files) or code without return values, and requires instruction-tuned LLMs."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluation responses from the 10 participants are not made available. Only aggregate Pass@1 results are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the human evaluation data collection: 140 ambiguous problems identified, questionnaires with three elements, each problem assessed by three participants."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes participants: 'ten participants, including three Ph.D. students, two Master's students, two senior researchers, and three industry developers' with Python experience details."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from requirement input through ambiguity detection (140 of 427 identified as ambiguous), question generation (avg 2.85 questions per problem), human response collection, to code generation is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed, including one author from Huawei Central Software Institute."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. One author is from Huawei, which has commercial interest in code generation tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for ChatGPT (gpt-3.5-turbo) or GPT-4 (gpt-4-turbo), despite evaluating on public benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 first threat discusses data leakage: 'Since these LLMs are trained on open-source code repositories, it is possible that some public benchmarks were included in their training data.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While Section 6.3 acknowledges the risk, the mitigation is weak — stating that benchmarks are 'manually crafted' and 'widely employed' does not address whether they appeared in training data. HumanEval (2021) and MBPP (2021) predate GPT-4's training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration mentioned for the human evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval mentioned despite involving 10 human participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 reports participant roles (PhD students, Master's students, senior researchers, industry developers) and Python experience levels (at least 3 years, 6 with 5+ years)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No explicit inclusion/exclusion criteria stated. Participants 'have at least three years of experience in Python development' is mentioned but it is unclear if this was a selection criterion."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The assignment of 42 problems per participant is mentioned but no randomization procedure is described for problem assignment."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding described. Participants answering clarifying questions likely knew they were evaluating ClarifyGPT."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether all 10 participants completed all assigned problems or if there was any dropout."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency reported despite ClarifyGPT requiring multiple LLM calls per problem (sampling n solutions, generating questions, simulating answers, final generation)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API spend or computational budget reported."
       }
     }
   },
   "claims": [
     {
       "claim": "ClarifyGPT elevates GPT-4 Pass@1 on MBPP-sanitized from 70.96% to 80.80% with human feedback.",
       "evidence": "Table 2 in Section 5.1 shows this result from human evaluation with 10 participants.",
       "supported": "moderate"
     },
     {
       "claim": "ClarifyGPT (Simulated Feedback) improves GPT-4 average Pass@1 across four benchmarks from 68.02% to 75.75%.",
       "evidence": "Table 3 in Section 5.2 shows results across HumanEval, HumanEval-ET, MBPP-sanitized, MBPP-ET.",
       "supported": "moderate"
     },
     {
       "claim": "ClarifyGPT (Simulated Feedback) improves ChatGPT average Pass@1 from 58.55% to 67.22%.",
       "evidence": "Table 3 in Section 5.2.",
       "supported": "moderate"
     },
     {
       "claim": "ClarifyGPT demonstrates robustness to the number of demonstrations, consistently outperforming Default from zero-shot to three-shot.",
       "evidence": "Table 4 in Section 5.3 shows consistent improvements, though zero-shot gains are marginal (0.0-2.8%).",
       "supported": "moderate"
     },
     {
       "claim": "The user simulation method produces high-fidelity simulated feedback close to real user responses.",
       "evidence": "Comparison of ClarifyGPT (Human Feedback) vs (Simulated Feedback) in Table 3 shows similar but not identical results. Only tested on MBPP-sanitized/MBPP-ET overlap.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "ClarifyGPT improves LLM code generation by detecting ambiguous requirements via code consistency checking and generating targeted clarifying questions. With human feedback, it improves GPT-4 Pass@1 by 13.87% on MBPP-sanitized. With simulated feedback, it achieves average improvements of 11.52% (GPT-4) and 15.07% (ChatGPT) across four benchmarks. The framework outperforms GPT-Engineer, which asks questions indiscriminately for all requirements.",
   "red_flags": [
     {
       "flag": "No statistical tests or variance reporting",
       "detail": "Despite running experiments three times, no standard deviations, confidence intervals, or significance tests are reported. It is impossible to assess whether observed differences are statistically meaningful."
     },
     {
       "flag": "Simulated feedback uses ground-truth test cases",
       "detail": "The user simulation method provides ground-truth test cases to the LLM to generate simulated answers. This gives the simulation access to information real users would not have, potentially inflating performance in the automated evaluation."
     },
     {
       "flag": "No inference cost analysis",
       "detail": "ClarifyGPT requires multiple LLM calls per problem (sampling n code solutions, generating test inputs, generating questions, simulating answers, final generation) but reports no cost or latency comparison with baselines."
     },
     {
       "flag": "Small human evaluation sample",
       "detail": "Only 10 participants with no power analysis, no IRB approval, and no randomization or blinding described. Results may not generalize."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational for LLM code generation evaluation."
     },
     {
       "title": "Self-collaboration Code Generation via ChatGPT",
       "authors": ["Yihong Dong", "Xue Jiang", "Zhi Jin", "Ge Li"],
       "year": 2023,
       "arxiv_id": "2304.07590",
       "relevance": "Multi-agent collaboration approach for LLM code generation."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Introduces extended test suites (HumanEval-ET, MBPP-ET) and type-aware mutation used by ClarifyGPT."
     },
     {
       "title": "CodeT: Code Generation with Generated Tests",
       "authors": ["Bei Chen"],
       "year": 2022,
       "arxiv_id": "2207.10397",
       "relevance": "Uses generated tests to improve code generation quality, related post-processing approach."
     },
     {
       "title": "CODAMOSA: Escaping coverage plateaus in test generation with pre-trained large language models",
       "authors": ["Caroline Lemieux", "Jeevana Priya Inala", "Shuvendu K Lahiri", "Siddhartha Sen"],
       "year": 2023,
       "relevance": "LLM-augmented test generation for software engineering."
     },
     {
       "title": "Python Code Generation by Asking Clarification Questions",
       "authors": ["Haau-Sing Li", "Mohsen Mesgar", "André F. T. Martins", "Iryna Gurevych"],
       "year": 2023,
       "relevance": "Most directly related prior work on clarification questions for code generation."
     },
     {
       "title": "Interactive Code Generation via Test-Driven User-Intent Formalization",
       "authors": ["Shuvendu K. Lahiri"],
       "year": 2022,
       "arxiv_id": "2208.05950",
       "relevance": "Interactive code generation approach using test-driven intent formalization."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei"],
       "year": 2022,
       "relevance": "Foundational prompting technique used as both inspiration and baseline in this work."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used in evaluation."
     }
   ]
 }