ai-research-survey

scan-v5.json (18629B)

---

 {
   "scan_version": 5,
   "paper_type": "survey",
   "paper": {
     "title": "Designing Empirical Studies on LLM-Based Code Generation: Towards a Reference Framework",
     "authors": [
       "Nathalia Nascimento",
       "Everton Guimaraes",
       "Paulo Alencar"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.03862"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims grounding in prior experience ([8,11,12]) and comparative analysis are supported by Section 3's documented search (75 papers, 32 retained, 13 analyzed).",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a framework-design paper, not an empirical study making causal claims about experimental outcomes.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Framework explicitly scoped to 'LLM-based code generation' studies. Section 8 acknowledges future extension to other SE tasks, defining current boundaries.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Framework-design paper with no empirical claims requiring alternative explanations.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "No empirical claims about measured vs. claimed outcomes; framework paper.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated Limitations or Threats-to-Validity section. Section 8 (Future Plans) acknowledges framework needs refinement but doesn't formally assess current limitations.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": false,
         "answer": false,
         "justification": "Framework-design paper without empirical threats.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Framework explicitly scoped to LLM-based code generation (title, abstract, introduction). Future extension to other SE tasks is mentioned, defining current boundaries.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section or statement present.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors' institutional affiliations clearly listed (Penn State, Waterloo).",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding disclosed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement provided.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Framework components (Problem Sources, Quality Attributes, Metrics, Environment, etc.) explicitly defined in Section 5. Quality attributes grounded in ISO/IEC 25010.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Abstract and introduction explicitly state: 'we propose a theoretical framework for designing and reporting empirical studies on LLM-based code generation.' Contribution is unambiguous.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 systematically contrasts this work with Schneider et al., Yeo et al., De Martino et al., and Wagner et al., showing how this framework differs (e.g., 'our approach provides a structured, bottom-up framework').",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "survey": {
       "search_and_selection": {
         "search_strategy_reproducible": {
           "applies": true,
           "answer": true,
           "justification": "Exact boolean search string provided: '((LLM OR LLMs...) AND (\"code generation\"...) AND (empirical AND (compar* OR...)))' in ACM Digital Library.",
           "source": "haiku"
         },
         "inclusion_exclusion_explicit": {
           "applies": true,
           "answer": true,
           "justification": "Stated explicitly: included 'empirical evaluations of LLMs on code generation tasks'; excluded 'education, user perception, tasks unrelated to code generation, non-empirical position/vision papers.'",
           "source": "haiku"
         },
         "prisma_or_structured_protocol": {
           "applies": true,
           "answer": false,
           "justification": "No PRISMA checklist, Cochrane protocol, or structured systematic review methodology cited. Approach is ad hoc.",
           "source": "haiku"
         },
         "search_terms_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full search string provided in Section 3 with all boolean operators and field specifications.",
           "source": "haiku"
         },
         "databases_listed": {
           "applies": true,
           "answer": true,
           "justification": "ACM Digital Library explicitly named. Only one database searched, limiting comprehensiveness.",
           "source": "haiku"
         },
         "screening_process_documented": {
           "applies": true,
           "answer": true,
           "justification": "Screening counts documented: 75 initial → 32 retained → 13 analyzed (11 most-cited + 2 snowballed). Counts provided but filtering methodology is sparse.",
           "source": "haiku"
         },
         "review_scope_justified": {
           "applies": true,
           "answer": false,
           "justification": "No justification for 2023-2025 date range, single-database scope, or why ACM-only (ignoring arXiv, IEEE, others in the field). Scope is stated but not reasoned.",
           "source": "haiku"
         }
       },
       "synthesis_quality": {
         "conflicting_findings_acknowledged": {
           "applies": false,
           "answer": false,
           "justification": "Framework distillation paper, not a synthesis of empirical findings across papers. No discussion of conflicting results or disagreements in the literature.",
           "source": "haiku"
         },
         "quality_assessment_of_sources": {
           "applies": true,
           "answer": false,
           "justification": "No quality rubric, risk-of-bias assessment, or structured evaluation of the 13 papers analyzed. Selection criterion was 'most cited papers' without methodological appraisal.",
           "source": "haiku"
         },
         "publication_bias_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of publication bias, selection effects, negative results, or how 'most cited' criterion may distort the sample.",
           "source": "haiku"
         },
         "quantitative_synthesis_present": {
           "applies": true,
           "answer": false,
           "justification": "No meta-analysis, vote counting, effect-size aggregation, or quantitative synthesis. Pure qualitative framework extraction.",
           "source": "haiku"
         },
         "recommendations_supported_by_evidence": {
           "applies": true,
           "answer": false,
           "justification": "Framework components are grounded in the 13 papers' practices but no evidence is provided that using these components improves study quality. Prescriptive but not evidence-based.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Empirical evaluation of LLM-based code generation lacks standardization, with studies varying widely in goals, tasks, and metrics.",
       "evidence": "Section 1 identifies fragmentation: 'Studies often adopt ad hoc experimental setups, resulting in limited reproducibility, poor comparability.' Authors cite Baltes et al. on unique LLM challenges (non-determinism, version drift, transparency).",
       "supported": "strong"
     },
     {
       "claim": "A bottom-up framework distilled from existing literature can organize core elements of LLM code generation experiments.",
       "evidence": "Section 3 documents search (75 papers → 32 → 13 analyzed). Section 5 identifies six framework components (Coding Task, Quality/Metrics, Empirical Research, Environment, LLM Model, Generated Output) recurring across studies.",
       "supported": "moderate"
     },
     {
       "claim": "The framework is applicable to diverse empirical setups.",
       "evidence": "Section 6 maps two representative papers (Ouyang et al., Ren et al.) to framework components, showing how it generalizes. But only 2 validation cases are presented.",
       "supported": "weak"
     },
     {
       "claim": "Domain-specific quality attributes (correctness, efficiency, bias, security) are critical to LLM code evaluation.",
       "evidence": "Section 5.3 cites ISO/IEC 25010 and empirical literature to group quality concerns into Functional, Technical, Resource Efficiency, and Ethical/Social categories with examples from [3, 5, 9, 11, 14, 18, 19, 22].",
       "supported": "strong"
     },
     {
       "claim": "The framework will evolve into an automated tool for research protocol generation.",
       "evidence": "Section 8 outlines future plans: 'automatic design of research protocols' where researchers specify domain and GQM, and the tool recommends questions, metrics, and study design. This is a prospective claim, not validated.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "case-study"
   ],
   "key_findings": "The paper proposes a six-component framework for standardizing empirical studies on LLM-based code generation (Coding Task, Quality/Metrics, Empirical Research, Environment, LLM Model, Generated Output), derived from a search of 75 papers (32 retained, 13 analyzed). The framework identifies recurring elements (problem sources like LeetCode/GitHub, quality attributes like correctness and efficiency, comparative methods) and gaps (non-determinism, prompt chaining, specification adherence) in the literature. Two validation mappings (Ouyang et al., Ren et al.) demonstrate applicability.",
   "red_flags": [
     {
       "flag": "Small analytical sample",
       "detail": "Only 13 of 32 retained papers analyzed for framework construction (11 most-cited + 2 snowballed). Risk of citation bias and non-representative sample."
     },
     {
       "flag": "Framework grounded in authors' own work",
       "detail": "Framework explicitly grounded in authors' prior papers [8, 11, 12]. Potential self-selection bias; framework components may over-represent authors' methodological choices."
     },
     {
       "flag": "Limited validation",
       "detail": "Only 2 papers used to validate framework applicability (Ouyang et al., Ren et al.). Insufficient evidence that framework generalizes broadly."
     },
     {
       "flag": "No quality assessment of source papers",
       "detail": "Source papers selected by citation count, not methodological quality. Framework may enshrine poor practices if high-citation papers have weak designs."
     },
     {
       "flag": "No inter-rater reliability",
       "detail": "No evidence that multiple reviewers independently extracted framework components from papers and achieved agreement. Single-rater framework construction."
     },
     {
       "flag": "Missing limitations section",
       "detail": "No formal Limitations section. Authors acknowledge in Section 8 that analysis is 'preliminary' but do not list current framework limitations."
     },
     {
       "flag": "Single-database search",
       "detail": "ACM Digital Library only. Excludes arXiv, IEEE Xplore, Scopus, Google Scholar. Risk of venue bias (may miss domain-specific venues)."
     }
   ],
   "cited_papers": [
     {
       "title": "Guidelines for Empirical Studies in Software Engineering involving Large Language Models",
       "relevance": "Wagner et al. proposes guidelines for LLM empirical study design; this framework complements by providing structural components."
     },
     {
       "title": "A Reference Model for Empirically Comparing LLMs with Humans",
       "relevance": "Schneider et al. addresses human-vs-LLM comparisons; this framework generalizes beyond human baselines."
     },
     {
       "title": "Framework for evaluating code generation ability of large language models",
       "relevance": "Yeo et al. proposes task taxonomy and metrics; this framework emphasizes experimental design structure."
     },
     {
       "title": "An Empirical Study of the Non-Determinism of ChatGPT in Code Generation",
       "relevance": "Ouyang et al. identifies non-determinism as underexplored; framework validation case demonstrates stability attribute integration."
     },
     {
       "title": "From Misuse to Mastery: Enhancing Code Generation with Knowledge-Driven AI Chaining",
       "relevance": "Ren et al. demonstrates prompt chaining for exception handling; framework validation case shows gaps in capturing advanced prompting strategies."
     },
     {
       "title": "RMCBench: Benchmarking Large Language Models' Resistance to Malicious Code",
       "relevance": "Chen et al. addresses security/robustness in code generation; exemplifies Ethical/Social quality attribute."
     },
     {
       "title": "Security Weaknesses of Copilot-Generated Code in GitHub Projects: An Empirical Study",
       "relevance": "Fu et al. evaluates security risks in generated code; demonstrates need for security quality metrics."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Framework is intended to guide empirical study design, but practical applicability limited by preliminary nature and lack of tool/template instantiation."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Proposes bottom-up framework approach vs. top-down guidelines, but conclusions (fragmentation exists, standardization needed) are widely acknowledged in the literature."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Mentions security/bias as quality attributes but does not raise novel safety concerns. Risk discussion is taxonomic, not alarm-raising."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, debate, or competing viewpoints presented. Consensual framework design paper."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Framework is abstract conceptual structure. No interactive tool, no runnable code, no live demo. Cannot be 'tried now.'"
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Penn State and Waterloo (mid-tier institutions). No Nobel laureates or household-name labs. Venues are arXiv (not yet peer-reviewed) and prior CASCON/MSR (mid-tier)."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "37862039",
         "title": "PeaTMOSS: Mining Pre-Trained Models in Open-Source Software",
         "points": 23,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=37862039",
         "created_at": "2023-10-12T19:35:57Z"
       },
       {
         "hn_id": "42333823",
         "title": "Show HN: Data Connector – Chat with Your Database and APIs",
         "points": 17,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42333823",
         "created_at": "2024-12-05T23:00:20Z"
       },
       {
         "hn_id": "45857764",
         "title": "Tidally Torn: Why the Most Common Stars May Lack Large, Habitable-Zone Moons",
         "points": 8,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45857764",
         "created_at": "2025-11-08T16:18:41Z"
       },
       {
         "hn_id": "46210641",
         "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code",
         "points": 4,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=46210641",
         "created_at": "2025-12-09T21:05:49Z"
       },
       {
         "hn_id": "46194269",
         "title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46194269",
         "created_at": "2025-12-08T16:29:33Z"
       },
       {
         "hn_id": "42535956",
         "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42535956",
         "created_at": "2024-12-28T23:45:41Z"
       },
       {
         "hn_id": "45683970",
         "title": "Parse: LLM Driven Schema Optimization for Reliable Entity Extraction",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=45683970",
         "created_at": "2025-10-23T16:42:00Z"
       },
       {
         "hn_id": "47021638",
         "title": "To ReAct or not to ReAct?",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47021638",
         "created_at": "2026-02-15T06:57:48Z"
       },
       {
         "hn_id": "46200850",
         "title": "Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46200850",
         "created_at": "2025-12-09T03:13:01Z"
       },
       {
         "hn_id": "43050120",
         "title": "Understanding Workers' Internal and External Representations of Complex Data",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43050120",
         "created_at": "2025-02-14T16:31:31Z"
       }
     ],
     "top_points": 23,
     "total_points": 63,
     "total_comments": 2
   }
 }