ai-research-survey

scan.json (20528B)

---

 {
   "paper": {
     "title": "Designing Empirical Studies on LLM-Based Code Generation: Towards a Reference Framework",
     "authors": ["Nathalia Nascimento", "Everton Guimaraes", "Paulo Alencar"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.03862"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis", "theoretical"],
   "key_findings": "The paper proposes a bottom-up theoretical framework for designing empirical studies on LLM-based code generation, organized into six core components: Coding Task, Quality and Metrics Evaluation, Empirical Research, Environment, LLM Model, and Generated Output. The framework was grounded in analysis of 13 papers (11 most cited from a 32-paper dataset plus 2 snowballed), with 9 used for construction and 2 for validation. The validation mappings revealed extension opportunities such as formalizing non-determinism and prompt chaining as framework elements.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or repository URL is provided. The framework is presented only as a figure and textual description."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references a publicly available curated bibliography dataset at doi:10.5281/zenodo.17230476 (reference [1])."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. The work is primarily conceptual but could have specified any analysis tooling."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided for replicating the framework construction or paper selection process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical framework paper with no quantitative experiments or statistical results."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative quantitative claims are made that would require significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative experiments are conducted; the paper proposes a conceptual framework."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experimental sample sizes to justify; this is a framework paper."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs producing variance; this is a theoretical/conceptual contribution."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 (Related Work) compares against four prior frameworks: Schneider et al. [17], Yeo et al. [23], De Martino et al. [4] (PRIMES), and Wagner et al. [20], explaining how the proposed framework differs from each."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All compared frameworks are from 2024-2025, which is contemporary to this 2025 paper."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The framework is a conceptual artifact with no separable components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative evaluation of the framework is performed; validation is done via qualitative case mappings."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation or expert review of the framework's utility or completeness is reported. The validation consists only of the authors mapping two studies to the framework."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 uses 2 randomly selected studies that were not used in framework construction as held-out validation instances."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-component breakdowns of how each validation instance maps to the framework, plus extension opportunities per component."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The validation instances reveal gaps — elements not covered by the framework (e.g., stability as a quality attribute, prompt chaining under prompt engineering), which are discussed as extension opportunities in Section 6 and Table 1."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies framework limitations through the validation mappings — elements that the framework does not yet cover, such as non-determinism analysis and specification conformance metrics."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the framework 'organizes evaluation around core components' and they 'demonstrate its applicability through representative case mappings.' Both are supported by Figure 1, Table 1, and Section 6."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It proposes a framework and demonstrates applicability through case mappings."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Towards a Reference Framework' suggesting generality, but the framework is grounded in only 9 papers from a single database (ACM DL), with validation on only 2 instances. The paper does not explicitly bound the generalization to LLM code generation studies found in ACM DL."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative framework designs, whether different paper selections would yield different components, or whether the bottom-up approach introduces selection bias."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the framework supports 'standardized and comprehensive experimentation' but measures applicability only through two author-performed case mappings, without discussing the gap between mapping coverage and actual standardization utility."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No LLM models are used in this study; it is a framework proposal paper."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this study."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No model inference or training is performed."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 3 provides the search string and states 75 papers were retrieved, 32 retained after inclusion/exclusion, and 13 selected (11 most cited + 2 snowballed). However, the actual filtering criteria for reducing 75 to 32 are stated only generically. The criteria for 'most cited' selection threshold and snowballing source are not detailed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper moves from Section 6 (framework instances) directly to Section 7 (Conclusion) and Section 8 (Future Plans) without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the framework does NOT cover or what settings it should NOT be applied to, beyond vague mentions of future extension to other SE tasks in Section 8."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The curated bibliography dataset is available at doi:10.5281/zenodo.17230476 (reference [1]), described as 'publicly available' with 'selection justifications.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the search string, database (ACM DL), time range (2023-2025), and selection process (75 retrieved, 32 retained, 13 selected)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data source is a literature search of a standard database."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from 75 to 32 papers lacks detail on how many were excluded at each criterion. The further reduction from 32 to 13 is described as '11 most cited + 2 snowballed' but without stating the citation threshold or snowballing procedure."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Pennsylvania State University and University of Waterloo."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model on a benchmark. It is a framework proposal based on literature analysis."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation on benchmarks is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation on benchmarks is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical framework paper; no inference or computation is performed."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical framework paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "Section 3 describes a structured search with a boolean query and inclusion/exclusion criteria, but there is no PRISMA flow diagram, no protocol registration, and the filtering steps lack the detail needed for a reproducible systematic protocol. Only 13 of 32 retained papers were analyzed, selected by citation count rather than a systematic criterion."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not assess the methodological quality of the 13 source papers it analyzes. All papers are treated equally in extracting framework components regardless of their rigor."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The selection of '11 most cited papers' actively introduces citation bias (well-cited ≠ rigorous), and this is not acknowledged."
       }
     }
   },
   "claims": [
     {
       "claim": "Empirical evaluation of LLM-based code generation lacks standardization, with studies varying widely in goals, tasks, and metrics.",
       "evidence": "Stated in abstract and Section 1 as motivation, supported by the diversity shown across the 13 analyzed papers in Sections 4-5.",
       "supported": "moderate"
     },
     {
       "claim": "The proposed framework organizes evaluation around six core components and supports structured experimentation.",
       "evidence": "Figure 1 presents the framework structure. Sections 4-5 describe the components. The framework is grounded in analysis of 9 papers and validated on 2 held-out papers (Section 6, Table 1).",
       "supported": "moderate"
     },
     {
       "claim": "The framework's applicability is demonstrated through representative case mappings that reveal extension opportunities.",
       "evidence": "Table 1 maps two held-out papers (Ouyang et al. [14] and Ren et al. [16]) to framework components, identifying gaps such as non-determinism and prompt chaining. However, only 2 validation instances from a narrow sample limits the strength of this evidence.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Self-citation dominance",
       "detail": "Three of the authors' own papers [8, 11, 12] (plus [13]) are among the 13 papers used to construct and ground the framework. The framework is partly built on the authors' own experimental patterns, which could bias the framework toward their practices rather than community-wide patterns."
     },
     {
       "flag": "Tiny validation sample",
       "detail": "The framework is validated on only 2 papers, both from the same 32-paper dataset. This is insufficient to claim generalizability. The paper acknowledges this only implicitly through future work plans."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity discussion despite being a methodology paper about empirical rigor. This is a significant omission given the paper's own subject matter."
     },
     {
       "flag": "Citation-count selection bias",
       "detail": "Selecting the '11 most cited papers' for framework construction introduces bias toward well-known, mainstream approaches. Novel or minority experimental designs are systematically excluded. This is not acknowledged."
     },
     {
       "flag": "Single database search",
       "detail": "The search was conducted only in ACM Digital Library, excluding IEEE Xplore, arXiv, Scopus, and other major venues where LLM code generation papers are published."
     }
   ],
   "cited_papers": [
     {
       "title": "Guidelines for Empirical Studies in Software Engineering involving Large Language Models",
       "authors": ["Sebastian Baltes", "Florian Angermeir", "Chetan Arora"],
       "year": 2025,
       "relevance": "Meta-research on empirical methodology for LLM studies in software engineering."
     },
     {
       "title": "RMCBench: Benchmarking Large Language Models' Resistance to Malicious Code",
       "authors": ["Jiachi Chen", "Qingyuan Zhong", "Yanlin Wang"],
       "year": 2024,
       "doi": "10.1145/3691620.3695480",
       "relevance": "Benchmark for evaluating LLM safety in code generation contexts."
     },
     {
       "title": "Security Weaknesses of Copilot-Generated Code in GitHub Projects: An Empirical Study",
       "authors": ["Yujia Fu", "Peng Liang", "Amjed Tahir"],
       "year": 2025,
       "doi": "10.1145/3716848",
       "relevance": "Empirical study of security vulnerabilities in AI-generated code."
     },
     {
       "title": "On the Effectiveness of Large Language Models in Domain-Specific Code Generation",
       "authors": ["Xiaodong Gu", "Meng Chen", "Yalan Lin"],
       "year": 2025,
       "doi": "10.1145/3697012",
       "relevance": "Empirical evaluation of LLMs across code generation domains."
     },
     {
       "title": "Bias Testing and Mitigation in LLM-based Code Generation",
       "authors": ["Dong Huang", "Jie M. Zhang", "Qingwen Bu"],
       "year": 2025,
       "doi": "10.1145/3724117",
       "relevance": "Addresses bias in LLM code generation, relevant to evaluation methodology."
     },
     {
       "title": "An Empirical Study of the Non-Determinism of ChatGPT in Code Generation",
       "authors": ["Shuyin Ouyang", "Jie M. Zhang", "Mark Harman"],
       "year": 2025,
       "doi": "10.1145/3697010",
       "relevance": "Studies non-determinism in LLM code generation — key methodological concern for reproducibility."
     },
     {
       "title": "SALLM: Security Assessment of Generated Code",
       "authors": ["Mohammed Latif Siddiq", "Joanna Cecilia da Silva Santos"],
       "year": 2024,
       "doi": "10.1145/3691621.3694934",
       "relevance": "Security assessment framework for LLM-generated code."
     },
     {
       "title": "A Reference Model for Empirically Comparing LLMs with Humans",
       "authors": ["Kurt Schneider", "Farnaz Fotrousi", "Rebekka Wohlrab"],
       "year": 2025,
       "relevance": "Reference model for LLM-vs-human empirical comparisons, directly related framework."
     },
     {
       "title": "Framework for evaluating code generation ability of large language models",
       "authors": ["Sangyeop Yeo", "Yu-Seung Ma", "Sang Cheol Kim"],
       "year": 2024,
       "relevance": "Evaluation framework for LLM code generation with task taxonomy."
     },
     {
       "title": "Towards evaluation guidelines for empirical studies involving LLMs",
       "authors": ["Stefan Wagner", "Marvin Muñoz Barón", "Davide Falessi", "Sebastian Baltes"],
       "year": 2025,
       "relevance": "Guidelines for empirical LLM studies — directly relevant meta-research."
     },
     {
       "title": "CoderUJB: An Executable and Unified Java Benchmark for Practical Programming Scenarios",
       "authors": ["Zhengran Zeng", "Yidong Wang", "Rui Xie"],
       "year": 2024,
       "doi": "10.1145/3650212.3652115",
       "relevance": "Unified benchmark for LLM code generation evaluation."
     }
   ]
 }