ai-research-survey

scan.json (27392B)

---

 {
   "paper": {
     "title": "Benchmarking Correctness and Security in Multi-Turn Code Generation",
     "authors": ["Ruchit Rawal", "Jeffrey Yang Fan Chiang", "Chihao Shen", "Jeffery Siyuan Tian", "Aastha Mahajan", "Tom Goldstein", "Yizheng Chen"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.13859"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Multi-turn code generation degrades correctness and security by 20-27% compared to single-turn, even for state-of-the-art models. The degradation is not explained by increased context length alone but by the challenge of reasoning over interdependent turns. Code-diff generation produces more insecure code than full-program generation. Agent scaffoldings (Aider, Codex, OpenHands) improve single-turn performance but are less effective in multi-turn scenarios.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We release our dataset here, and code here' in Section 1, and provides a HuggingFace link (https://huggingface.co/datasets/ai-sec-lab/mt-sec) in the abstract."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released on HuggingFace at https://huggingface.co/datasets/ai-sec-lab/mt-sec as stated in the abstract."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using NVIDIA A40 and A100 GPUs (Appendix B) but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The code is referenced but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-11 are reported as point estimates (e.g., '51.9% C&S') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., '20-27% drop') but reports no statistical significance tests. Differences are based on comparing raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute percentage drops with baselines (e.g., 'C&S score of Aider + GPT-5T decreases by 27.3%, falling from 53% to 25.7%'), providing sufficient context for effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 2,376 multi-turn tasks from 792 seed prompts, but no justification for why this size is sufficient or any power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread across runs is reported. Results appear to be single-run evaluations with zero temperature for non-reasoning models."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Single-turn performance serves as the primary baseline. The MT-Random control condition is also included. 32 models and 3 agent frameworks are compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models like GPT-5, Claude Opus 4, O4-Mini, Gemini 2.5 Pro, and DeepSeek-R1, which are state-of-the-art at time of evaluation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation studies are provided: MT-Random control (Section 4), security policy insertion points (Table 2), Aider agent component ablation (Table 10), and editing format comparison (Table 11)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two primary metrics: Correct & Secure (C&S) and Correct & Insecure (C&I), and also reports aggregated correctness (C&S + C&I)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human verification by three security experts evaluated task faithfulness (93.1% accepted) and interaction-type alignment (91.6% agreement) of the generated benchmark tasks (Section 3)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark reuses dynamic correctness and security tests from the seed single-turn datasets (SecCodePLT, BaxBench), which are separate from the generation prompts."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by interaction type (expansion, editing, refactor), by model, and per-CWE analysis is mentioned. Table 1 provides full per-model breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive qualitative failure analysis is provided in Appendix D (forgetting security instructions, over-refusals) and Appendix F (agent failure modes for OpenHands, Codex, Aider)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key negative results include: agent scaffoldings underperform in multi-turn (despite improving single-turn), security policies don't close the single-to-multi-turn gap, every-turn policy can hurt performance, and over-refusals in thinking models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (20-27% C&S drop, code-diffs less secure, agents less effective in multi-turn) are all supported by Tables 1, 3, and surrounding analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The MT-Random control experiment (Fig. 3) provides a controlled comparison to isolate the effect of multi-turn reasoning from context length, supporting the causal claim that degradation comes from cross-turn dependencies rather than context length alone."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Multi-Turn Code Generation' broadly but results are limited to tasks derived from two specific benchmarks (SecCodePLT, BaxBench) covering 27 CWEs. The paper does not bound generalizations to these specific task types or acknowledge that real-world multi-turn coding may differ from synthetic transformations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly tests alternative explanations: context length (MT-Random experiment, Section 4), target task length (Appendix C.1), and provides the longer single-turn baseline comparison (Table 6)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures pass/fail on dynamic unit tests and frames this as evaluating 'correctness and security' in 'real-world coding workflows.' It does not discuss the gap between passing unit tests and actual real-world security, nor acknowledge that their synthetic multi-turn tasks are a proxy for actual developer workflows."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 in Appendix A provides exact model checkpoints for all 32+ models (e.g., 'gpt-5-2025-08-07', 'claude-opus-4-20250514', 'o4-mini-2025-04-16')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates for multi-turn generation (editing, expansion, refactor) are provided in Appendix G (Sections G.1-G.4), including the targeted regeneration prompt."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states: zero temperature for non-reasoning models, 0.7 for reasoning models, thinking budget set to 'low' or 4000 tokens. Agent configurations are detailed in Appendix E."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Agent scaffolding is described in detail in Appendix E.1 for all three agents (OpenHands, Codex, Aider), including versions, configurations, and interaction setup."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The seed prompt selection criteria are documented (Section 3): dynamic tests required, longest implementations selected, 22-24 per CWE for SecCodePLT. Consistency guardrails and human verification processes are detailed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion & Conclusions section (Section 5) discusses limitations including lack of intermediate turn evaluation and quality fluctuation during turns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations discussed are generic ('we do not evaluate the quality of intermediate code') rather than specific threats like the validity of synthetic multi-turn transformations as a proxy for real developer interactions, or selection bias in CWE coverage."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to the specific synthetic transformation approach, the two source benchmarks, or the specific CWEs tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released on HuggingFace (https://huggingface.co/datasets/ai-sec-lab/mt-sec) for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full data construction pipeline: seed prompt selection, synthetic dialogue generation using GPT-4o, consistency guardrails, and human verification."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For human verification, the paper states 'three security experts (two authors and one external volunteer)' but does not describe how the external volunteer was recruited or whether this introduces bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage pipeline is documented: seed prompt selection (with counts), synthetic generation with guardrails (up to 3 regenerations), and human verification with acceptance rates (93.1% faithfulness, 91.6% alignment)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists funding: Open Philanthropy, NSF CAREER Award CNS-2442719, DARPA TIAMAT, NSF TRAILS Institute, Capital One Bank, Amazon Research Award, Google DeepMind, and OpenAI."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with University of Maryland, College Park, as stated in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The paper received gifts from Google DeepMind and OpenAI, whose models (Gemini, GPT series) are evaluated. These funders have a financial interest in how their models perform on benchmarks."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 32 models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is constructed from SecCodePLT and BaxBench, which are public datasets. No discussion of whether these appeared in model training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SecCodePLT and BaxBench are publicly available benchmarks that may have been in training data for models evaluated. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in an experimental study. The human verification of benchmark quality is not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The paper is a benchmark evaluation of LLMs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in an experimental study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in an experimental study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in an experimental study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in an experimental study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in an experimental study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'substantial cost of running evaluations' as a reason for limiting BaxBench experiments, but does not report actual API costs, tokens consumed, or per-example costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states: 'two NVIDIA A40 GPUs, each with 48GB of memory, and two NVIDIA A100 GPUs, each with 82GB of memory' for open-source model experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be single-run with zero temperature (deterministic) for non-reasoning models. No seed sensitivity analysis is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state the number of runs per evaluation. Appendix B mentions 'we generate one sample per environment-task' for BaxBench but does not clarify for SecCodePLT."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Temperature and thinking budgets appear to use defaults or recommended values without search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "For agent configurations, the paper uses default settings without justifying why these are optimal or comparing alternatives systematically (though Table 11 compares editing formats)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed, so no multiple comparison correction is applied despite comparing 32+ models across multiple conditions."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors create the benchmark and evaluate all models on it. No discussion of author-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of performance as a function of compute budget. Agent evaluations use different compute profiles without normalization."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The MT-Random experiment (Section 4, Fig. 3) validates that the benchmark measures multi-turn reasoning rather than just context length effects. The paper also discusses what the benchmark captures vs. real-world workflows."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Agent scaffoldings (Aider, OpenHands, Codex) are compared against base LLMs, but different agents use different scaffolding. When comparing agents, the scaffold-model confound is not explicitly addressed as a variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SecCodePLT and BaxBench are publicly available datasets. The paper does not discuss whether models trained after their publication may have seen solutions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through context or whether multi-turn prompts provide hints not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The three interaction types for each seed prompt share the same tests. The paper does not discuss whether results across interaction types are non-independent."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied despite using publicly available benchmarks with models that could have seen them."
       }
     }
   },
   "claims": [
     {
       "claim": "There is a consistent 20-27% drop in 'correct & secure' outputs from single-turn to multi-turn settings across state-of-the-art models.",
       "evidence": "Table 1 shows agent-scaffolded models experience 25-27% decline in C&S for MT-expansion, 14-17% for MT-editing, and 10-12% for MT-refactor. Non-agentic LLMs show 15-20% drops.",
       "supported": "strong"
     },
     {
       "claim": "Performance degradation is not solely due to longer context length.",
       "evidence": "MT-Random experiment (Fig. 3) shows that replacing first two turns with unrelated prompts (preserving context length) yields performance comparable to single-turn, while real multi-turn degrades substantially.",
       "supported": "strong"
     },
     {
       "claim": "Code-diff generation produces more incorrect and insecure code than full-program generation.",
       "evidence": "Table 3 shows consistent decline in C&S and increase in C&I across all 6 models tested when generating code-diffs vs full code (e.g., O4-Mini drops 10.5% in C&S).",
       "supported": "moderate"
     },
     {
       "claim": "Agent scaffoldings boost single-turn performance but are not as effective in multi-turn scenarios.",
       "evidence": "Table 9 shows agents achieve higher single-turn C&S (e.g., O4-Mini Agent: 68.8% vs LLM: 56.8%) but lower multi-turn C&S (Agent: 33.0% vs LLM: 38.7% for MT-expansion).",
       "supported": "strong"
     },
     {
       "claim": "Thinking models exhibit over-refusals in multi-turn settings that don't occur in single-turn.",
       "evidence": "Section 4 reports Claude Sonnet 4T has 2.7% over-refusal rate and O3T has 0.8% in multi-turn, with zero refusals in single-turn. Verified by regex-based heuristic plus manual inspection.",
       "supported": "moderate"
     },
     {
       "claim": "Security policy insertion in the final turn is most effective, sometimes outperforming every-turn insertion.",
       "evidence": "Table 2 shows final-turn insertion yields best C&S for OpenAI models (e.g., O3T: 34.8% final-turn vs 33.1% every-turn). Qualitative analysis in Appendix D.3 explains the mechanism.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims (20-27% drop, model rankings) are based on raw percentage comparisons without any significance tests, despite the potential for measurement noise."
     },
     {
       "flag": "No variance or multiple-run reporting",
       "detail": "Results appear to be single-run evaluations. Even with zero temperature, API-based models may have non-deterministic behavior, and no robustness checks are reported."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "The benchmark is derived from publicly available SecCodePLT and BaxBench. Models trained after these datasets' publication may have seen their solutions, potentially inflating single-turn scores and underestimating the true multi-turn gap."
     },
     {
       "flag": "Funding from evaluated companies",
       "detail": "The research received gifts from Google DeepMind and OpenAI, whose models are prominently evaluated. No competing interests statement is provided."
     },
     {
       "flag": "Synthetic multi-turn validity",
       "detail": "Multi-turn tasks are synthetically generated by GPT-4o transforming single-turn prompts. The paper assumes these transformations represent 'real-world coding workflows' but does not validate this against actual developer interaction logs."
     }
   ],
   "cited_papers": [
     {
       "title": "SecCodePLT: A Unified Platform for Evaluating the Security of Code GenAI",
       "authors": ["Yu Yang", "Yuzhou Nie", "Zhun Wang", "Yuheng Tang", "Wenbo Guo", "Bo Li", "Dawn Song"],
       "year": 2024,
       "arxiv_id": "2410.11096",
       "relevance": "Source benchmark for MT-Sec; evaluates correctness and security of LLM-generated code using dynamic tests."
     },
     {
       "title": "BaxBench: Can LLMs Generate Correct and Secure Backends?",
       "authors": ["Mark Vero", "Niels Mündler", "Victor Chibotaru", "Veselin Raychev", "Maximilian Baader", "Nikola Jovanović", "Jingxuan He", "Martin Vechev"],
       "year": 2025,
       "arxiv_id": "2502.11844",
       "relevance": "Source benchmark for MT-Sec; evaluates LLMs on self-contained backend application generation with security tests."
     },
     {
       "title": "SecRepoBench: Benchmarking LLMs for Secure Code Generation in Real-World Repositories",
       "authors": ["Connor Dilgren", "Purva Chiniya", "Luke Griffith", "Yu Ding", "Yizheng Chen"],
       "year": 2025,
       "arxiv_id": "2504.21205",
       "relevance": "Evaluates LLM secure code generation in repository-level contexts, extending beyond isolated tasks."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2025,
       "relevance": "Early foundational work evaluating security vulnerabilities in Copilot-generated code."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "One of three agent frameworks evaluated; open-source agent platform for software development tasks."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Influential agentic coding framework that relies on multi-turn interactions for software engineering tasks."
     },
     {
       "title": "Multi-IF: Benchmarking LLMs on Multi-Turn and Multilingual Instructions Following",
       "authors": ["Yun He"],
       "year": 2024,
       "arxiv_id": "2410.15553",
       "relevance": "Showed LLMs struggle with consistent instruction-following across multiple turns in NLP tasks."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroaki Hayashi"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "Prior multi-turn code generation benchmark that factorizes problems into sub-instructions."
     },
     {
       "title": "MINT: Evaluating LLMs in Multi-Turn Interaction with Tools and Language Feedback",
       "authors": ["Xingyao Wang", "Zihan Wang", "Jiateng Liu"],
       "year": 2023,
       "arxiv_id": "2309.10691",
       "relevance": "Multi-turn evaluation benchmark testing LLM problem-solving with tool and language feedback."
     },
     {
       "title": "CWEval: Outcome-Driven Evaluation on Functionality and Security of LLM Code Generation",
       "authors": ["Jinjun Peng", "Leyi Cui", "Kele Huang", "Junfeng Yang", "Baishakhi Ray"],
       "year": 2025,
       "arxiv_id": "2501.08200",
       "relevance": "Dynamic evaluation of LLM code generation security, showed static analyzers produce high false positive rates."
     },
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper referenced for model size-performance trends observed in the evaluation."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["Daya Guo"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Reasoning model evaluated in the benchmark; relevant to understanding thinking model performance patterns."
     }
   ]
 }