ai-research-survey

scan.json (28490B)

---

 {
   "paper": {
     "title": "Interactive Code Generation via Test-Driven User-Intent Formalization",
     "authors": [
       "Shuvendu K. Lahiri",
       "Sarah Fakhoury",
       "Aaditya Naik",
       "Georgios Sakkas",
       "Saikat Chakraborty",
       "Madanlal Musuvathi",
       "Jeevana Priya Inala",
       "Piali Choudhury",
       "Curtis von Veh",
       "Chenglong Wang",
       "Jianfeng Gao"
     ],
     "year": 2022,
     "venue": "arXiv",
     "arxiv_id": "2208.05950",
     "doi": "10.48550/arXiv.2208.05950"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TiCoder demonstrates that interactive test-driven code generation using LLM-generated tests can substantially improve pass@1 accuracy — by 22.49% on MBPP (48.24% → 70.73%) and 24.79% on HumanEval (30.49% → 55.28%) with a single simulated user query. Discriminative test ranking and dynamic test mutation are the most impactful components in ablation. The approach generates a user-accepted test within an average of 1.5–1.7 queries for 87–96% of benchmark examples.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided in the paper. Section VI mentions 'We aim to mitigate this by releasing model generated output in the near future' — a promise of future release, not actual release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "MBPP (sanitized) and HumanEval are public benchmarks, but the authors modified HumanEval by removing non-hidden input-output examples from docstrings (Section IV-B). The modified dataset and the cached Codex outputs are not released (only promised for future release in Section VI)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or software versions are provided beyond naming the Codex model."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The algorithmic description is detailed but there are no runnable scripts or reproduction guides."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 70.73%, 55.28%) with no confidence intervals or error bars on any figure or table."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims TiCoder 'outperforms' baselines based solely on comparing raw percentages (e.g., 70.73% vs 63.70% for CodeT on MBPP) without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage improvements with baseline context throughout. For example, Section V-A reports TiCoder pass@1@1 of 70.73% vs Codex baseline 48.24%, and the abstract quantifies improvements as '22.49% to 37.71%' absolute percentage points."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "MBPP has 427 examples and HumanEval has 164. These are standard benchmarks adopted without any discussion of whether these sample sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-C explicitly states: 'we only query Codex once to generate the initial code and test suggestions into a cache of Codex responses and refer to the same cache for all experiments.' Single-run results with no variance, standard deviation, or spread measures reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C and V-A compare against: Codex (t=0.8), Codex (t=0), a Baseline TiCoder variant (no mutation/ranking), CodeT, IdealRanking, and IdealTests upper bounds."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CodeT (Chen et al., 2022) and AlphaCode (Li et al., 2022) are contemporary works. Codex code-davinci-002 was state-of-the-art at the time of evaluation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C presents detailed ablation studies (Tables II and III) removing each component individually: code prompt, single assert, dynamic mutation, test ranking, and code ranking."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses pass@k@m for various k∈{1,2,5,10} and m∈{0,1,2,3,4,5}, plus accept@m (Section II-C, Table I). These measure different aspects: code correctness and test quality respectively."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using oracle simulation (reference implementations as proxy for user responses, Definition IV.1). Section VI acknowledges the lack of user study: 'We plan to conduct a user study to evaluate such metrics in practice.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are evaluated against hidden test sets from MBPP and HumanEval benchmarks (the ground truth tests Tp), which are separate from the LLM-generated tests used in the approach."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are shown per benchmark (MBPP vs HumanEval) but there is no breakdown by problem difficulty, problem type, or category within each benchmark."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure case analysis is presented. The paper does not discuss specific examples where TiCoder fails or where the approach breaks down, beyond the running example used for illustration."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Tables II-III) shows cases where removing components improves performance. For example, on HumanEval, removing the code prompt improves pass@1@1 from 55.27% to 59.62%. Section III-B notes that single-assert 'may also adversely impact the performance.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 22.49%–37.71% improvement on MBPP and 24.79%–53.98% on HumanEval are directly supported by Figures 5 and 6 in Section V."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are primarily in the ablation study (Section V-C), which uses controlled single-variable manipulation — removing one component at a time from the default configuration. This is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Interactive Code Generation via Test-Driven User-Intent Formalization' is broad, while results are limited to Python on two benchmarks using a single LLM (Codex). Section VI acknowledges limitations to 'different programs across different languages and problem domains,' but the title and framing extend well beyond the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats section (Section VI) discusses three threats but does not consider alternative explanations for the results — e.g., whether improvements come primarily from test-based filtering rather than the interactive component, or whether the oracle simulation inflates the benefit."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@k on hidden test suites and claims improvement in 'code generation accuracy.' This is a reasonably tight measurement-to-claim mapping. Section VI explicitly acknowledges the proxy gap of simulated vs. real user interaction: 'our automated evaluation assumes the user is able to answer the generated tests, and cannot account for the cognitive effort.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C specifies 'OpenAI's Codex code-davinci-002 model' — a specific model version identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows the full code and test prompt structures with concrete examples. Section III-A describes the prompt construction options in detail (pass vs choose(G) for test prompts). The prompts are template-based with clearly specified fill values."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C reports: 'temperature of 0.8 and a top p of 0.95,' maximum generation length of 300 tokens, 100 code suggestions, and 50 test suggestions."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. TiCoder is a pipeline-based algorithm (generate → mutate → rank → query) without autonomous agent behavior."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B describes using 'the sanitized version of the MBPP dataset' and explicitly describes the HumanEval modification: 'We modify the original HumanEval dataset to remove any (non-hidden) input-output examples that are included in the docstring.'"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats' is a dedicated section with substantive discussion of three specific threats to validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses three specific threats: (1) benchmarks may not represent real-world development, (2) Codex model stochasticity and endpoint discontinuation threatening replicability, (3) oracle simulation cannot account for user cognitive effort."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VI explicitly states: 'Our findings may not generalize to a different set of programs across different languages and problem domains.' It also notes the limitation to simulated rather than real user interaction."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The cached Codex outputs that all experiments build on are not released. Section VI states 'We aim to mitigate this by releasing model generated output in the near future' — future promise, not current availability."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C describes data collection: querying Codex with specific parameters (temperature 0.8, top-p 0.95, 100 code suggestions, 50 test suggestions per example), caching results, and using standard benchmark datasets."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses standard code generation benchmarks (MBPP and HumanEval) with simulated oracle interaction."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 and Section III detail the full pipeline: LLM query for codes (line 1) → LLM query for tests (line 2) → static mutation (line 3) → dynamic mutation (line 4) → interactive ranking loop (lines 6-18). The oracle simulation is defined precisely in Definition IV.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Microsoft Research (majority), University of Pennsylvania, and University of California San Diego. Internship relationships are noted."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The majority of authors are from Microsoft Research, which has a major investment in OpenAI (whose Codex model is evaluated). Microsoft has a financial interest in positive results for LLM-based code generation. This conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of Codex code-davinci-002's training data cutoff date. The paper does not state when the model's training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MBPP or HumanEval problems appeared in Codex's training data. MBPP was publicly available before Codex training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MBPP was published in 2021 and HumanEval was introduced alongside Codex. No discussion of whether benchmark solutions were in the training data, despite both benchmarks being publicly available."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All user interaction is simulated via oracle (reference implementation)."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Evaluation is fully automated using oracle simulation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The approach queries Codex for 100 code suggestions and 50 test suggestions per example, plus user interaction rounds. No API costs, latency, or per-example cost is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware details are provided despite substantial LLM API usage across 427 + 164 benchmark examples."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-C states: 'we only query Codex once to generate the initial code and test suggestions into a cache.' Single-run results with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C explicitly states they query Codex once and cache the results: 'we only query Codex once to generate the initial code and test suggestions into a cache of Codex responses and refer to the same cache for all experiments.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-C states 'We chose this configuration as default empirically as it performs the best on the pass@1@1 metric on the MBPP dataset,' but does not report how many configurations were searched or the computational cost of this search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C justifies the default configuration: 'We chose this configuration as default empirically as it performs the best on the pass@1@1 metric on the MBPP dataset.' The ablation study (Tables II-III) shows all alternative configurations tested."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many pairwise comparisons are made across configurations, metrics, and benchmarks with no statistical significance tests and therefore no multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare TiCoder against their own implementation of CodeT and other baselines without acknowledging the potential bias of evaluating their own system against self-implemented competitors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "TiCoder uses substantially more LLM queries than baseline Codex (additional test generation, mutation, and potentially multiple interaction rounds), but performance is never compared at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MBPP and HumanEval actually measure real-world code generation capability. The paper uses these benchmarks without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. TiCoder is an algorithmic pipeline, not an agentic scaffold. Comparisons are between algorithmic variants, not scaffold configurations."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MBPP (2021) and HumanEval (2021) were both published before Codex's training data was collected. No discussion of temporal leakage or whether solutions to these benchmarks were in the training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The oracle simulation provides perfect answers, which may inflate the perceived benefit compared to real user interaction."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MBPP or HumanEval problems share structural similarities with Codex's training data or with each other."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference, or decontamination analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "TiCoder improves pass@1 from 48.24% to 70.73% on MBPP and from 30.49% to 55.28% on HumanEval with a single simulated user query.",
       "evidence": "Section V-A, Figures 5a and 5b show pass@k@1 results across all baselines. The numbers are clearly presented in both figures and text.",
       "supported": "strong"
     },
     {
       "claim": "With 5 user queries, TiCoder achieves pass@1 of 85.95% on MBPP and 84.47% on HumanEval.",
       "evidence": "Section V-B, Figures 6a and 6b show pass@1@m for m=0 through 5 with clear progression.",
       "supported": "strong"
     },
     {
       "claim": "TiCoder generates a user-accepted test within an average of 1.7 queries for 87.12% of MBPP and 1.5 queries for 95.73% of HumanEval examples.",
       "evidence": "Section V-B, Figure 7 shows cumulative accept@m curves for both benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "Each component (test ranking, dynamic mutation, code ranking, static mutation, prompt design) contributes to TiCoder's effectiveness.",
       "evidence": "Section V-C, Tables II and III show ablation results. Test ranking and dynamic mutation are most impactful. However, the default configuration is not uniformly best across all metrics and both benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "TiCoder outperforms CodeT, which uses no user interaction, on MBPP with a single query and on HumanEval with 2 queries.",
       "evidence": "Section V-A shows TiCoder pass@1@1=70.73% vs CodeT 63.70% on MBPP. On HumanEval, TiCoder pass@1@1=55.28% is below CodeT's 58.54%, but with 2 queries TiCoder reaches 68.94% (Figure 6b).",
       "supported": "moderate"
     },
     {
       "claim": "Test mutation techniques improve the pool of tests over purely LLM-generated tests, shown by 10.72% (MBPP) and 13.74% (HumanEval) accuracy gain after 5 interactions.",
       "evidence": "Section V-B, Figures 6a and 6b compare IdealRanking (with mutations) vs BaselineIdeal (LLM-only tests with ideal ranking).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Simulated user evaluation only",
       "detail": "All experiments use an oracle (reference implementation) instead of real users. The cognitive cost and accuracy of real user responses is unknown. The paper acknowledges this but provides no user study, making the claimed 'interactive' benefit unvalidated with actual users."
     },
     {
       "flag": "No error bars or variance on any result",
       "detail": "All results come from a single cached Codex query. No repeated runs, no confidence intervals, no statistical tests. The stochasticity of Codex (acknowledged in Section VI) means results could vary substantially across runs."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Both MBPP and HumanEval were publicly available before Codex training. The high pass@100 values (89.70% MBPP, 90.85% HumanEval) could partly reflect memorization. No contamination check is performed."
     },
     {
       "flag": "Microsoft-OpenAI financial relationship",
       "detail": "The majority of authors are Microsoft Research employees evaluating OpenAI's Codex model. Microsoft is a major investor in OpenAI. This conflict of interest is not disclosed or acknowledged."
     },
     {
       "flag": "Discontinued model",
       "detail": "Codex code-davinci-002 was discontinued by OpenAI in March 2023. Results cannot be verified or replicated with the same model, and the cached outputs are not released."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced Codex and HumanEval benchmark, the primary model and one of two benchmarks used in this study."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced the MBPP benchmark used in this study for evaluating code generation."
     },
     {
       "title": "CodeT: Code generation with generated tests",
       "authors": ["Bei Chen", "Fengji Zhang", "Anh Nguyen"],
       "year": 2022,
       "relevance": "Primary baseline that uses LLM-generated tests to improve code ranking without user interaction."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "Uses test generation and code clustering for competitive programming — related approach to test-based code filtering."
     },
     {
       "title": "Productivity assessment of neural code completion",
       "authors": ["Albert Ziegler", "Eirini Kalliamvakou", "X. Alice Li"],
       "year": 2022,
       "relevance": "Evaluates Copilot productivity in real-world developer scenarios, directly relevant to AI-assisted coding evaluation."
     },
     {
       "title": "Expectation vs. experience: Evaluating the usability of code generation tools powered by large language models",
       "authors": ["Priyan Vaithilingam", "Tianyi Zhang", "Elena L. Glassman"],
       "year": 2022,
       "relevance": "User study showing developers struggle to understand LLM code suggestions, motivating the interactive approach."
     },
     {
       "title": "Do users write more insecure code with AI assistants?",
       "authors": ["Neil Perry", "Megha Srivastava", "Deepak Kumar", "Dan Boneh"],
       "year": 2022,
       "arxiv_id": "2211.03622",
       "relevance": "Studies security implications of AI-assisted coding, relevant to LLM code generation safety."
     },
     {
       "title": "CodaMosa: Escaping coverage plateaus in test generation with pre-trained large language models",
       "authors": ["Caroline Lemieux", "Jeevana Priya Inala", "Shuvendu K. Lahiri", "Siddhartha Sen"],
       "year": 2023,
       "relevance": "Uses LLMs to improve automated test generation, directly relevant to LLM-based testing capabilities."
     },
     {
       "title": "TOGA: A neural method for test oracle generation",
       "authors": ["Elizabeth Dinella", "Gabriel Ryan", "Todd Mytkowicz", "Shuvendu Lahiri"],
       "year": 2022,
       "relevance": "Neural approach to generating test oracles, relevant to LLM-based test generation for code verification."
     },
     {
       "title": "Adaptive test generation using a large language model",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2023,
       "arxiv_id": "2302.06527",
       "relevance": "LLM-based adaptive test generation, directly relevant to using LLMs for software testing."
     },
     {
       "title": "PaLM: Scaling language modeling with pathways",
       "authors": ["Aakanksha Chowdhery", "Sharan Narang", "Jacob Devlin"],
       "year": 2022,
       "relevance": "Major LLM with code generation capabilities, relevant to the landscape of code-generation models."
     },
     {
       "title": "A systematic evaluation of large language models of code",
       "authors": ["Frank F. Xu", "Uri Alon", "Graham Neubig", "Vincent J. Hellendoorn"],
       "year": 2022,
       "relevance": "Systematic evaluation of code LLMs including PolyCoder, relevant to benchmarking code generation models."
     },
     {
       "title": "Jigsaw: Large language models meet program synthesis",
       "authors": ["Naman Jain", "Skanda Vaidyanath", "Arun Iyer", "Nagarajan Natarajan"],
       "year": 2022,
       "relevance": "Uses user-provided tests to improve LLM code generation quality, closely related approach."
     }
   ]
 }