ai-research-survey

scan.json (20272B)

---

 {
   "paper": {
     "title": "From Requirements to Code: Understanding Developer Practices in LLM-Assisted Software Engineering",
     "authors": ["Jonathan Ullrich", "Matthias Koch", "Andreas Vogelsang"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2507.07548"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["qualitative"],
   "key_findings": "Practitioners do not use traditional requirements artifacts (user stories, functional requirements) as direct input for LLMs. Instead, they manually decompose requirements into programming tasks, enriching them with design decisions and architectural constraints before prompting. Three interaction patterns emerged: incremental code generation (pair programming style), manual coding with intelligent auto-completion, and extensive code generation. The study proposes a process model and content model for LLM-assisted implementation grounded in 18 practitioner interviews across 14 companies.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code is released. The paper provides an interview guide and codebook via a Zenodo replication package (https://zenodo.org/records/15005613), but no analysis code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is provided at Zenodo (https://zenodo.org/records/15005613) containing the interview guide and codebook with traces from in-vivo codes to themes."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study with no computational experiments requiring environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the interview guide and codebook are provided, there are no step-by-step instructions for reproducing the analysis. The coding process is described narratively but not as a reproducible protocol."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative interview study with no quantitative results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative comparisons are made; findings are qualitative themes derived from interviews."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative study; no effect sizes are applicable."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The authors state they stopped acquiring new interviewees once 'our theory was validated in subsequent interviews and no new themes emerged' after P15-P18, indicating theoretical saturation was reached."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative study with no quantitative experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative theory-building study, not a system evaluation. Baselines are not applicable."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No system evaluation requiring baselines."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative evaluation metrics; findings are qualitative."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate. The study itself is interview-based research, not a system evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No datasets or test sets involved."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides per-participant breakdowns by role, company size, application domain, requirements type, and interaction mode. Findings are organized by themes (process model, content model, interaction patterns)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where direct use of requirements failed: 'If I say GPT write a command interface for a Keithley, then I have in fact taken a requirement from my catalog and I've never actually had anything ready to use come out of it' (P10)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The core finding is a negative result: requirements as traditionally documented are too abstract for direct LLM input. Multiple participants report failed attempts at using requirements directly."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about requirements being too abstract, needing decomposition into programming tasks, and enrichment with design decisions are all supported by interview quotes in Section IV."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It proposes a descriptive theory of current practices based on qualitative interviews."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section III.D (External validity) acknowledges 'qualitative research does not aim for statistical generalization' and notes the sample's diversity as a mitigation, not a guarantee of generalizability."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed patterns. For instance, the decomposition pattern might reflect tool limitations at interview time rather than inherent necessity, but this is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully distinguishes what it measures (self-reported developer practices) from broader claims. It frames findings as 'a theory that describes processes' rather than claiming objective measurement of effectiveness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The study does not evaluate any AI models; it interviews practitioners about their practices."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No LLM prompting is performed as part of the research methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments requiring hyperparameter specification."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used in the research."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III.C describes the data analysis: exhaustive in-vivo coding producing 179 quotes, iterative clustering into themes, cross-validation by all authors, use of MaxQDA tool, export to Excel. Translation from German to English using DeepL is noted."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section III.D 'Threats to Validity' provides a substantive discussion covering internal, external, and construct validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses study-specific issues: recording and transcribing interviews to ensure accuracy, explaining study goals to participants to minimize misunderstandings, including diverse roles/industries, and cross-validating coding across all authors."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The threats-to-validity section discusses general methodological concerns but does not clearly delineate specific scope boundaries or excluded populations/settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw interview transcripts are not released, likely for participant privacy. Only the codebook and interview guide are in the replication package."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B describes interview conduct: 30-60 minute online interviews via Microsoft Teams, recorded, semi-structured guideline, two-part structure, all transcribed, German interviews translated via DeepL."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B states 'We applied convenience sampling using our network to acquire interview partners' and describes selection criteria (experience with code models in implementation phase, diverse roles and domains)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III.C documents the pipeline: interviews → transcription → in-vivo coding (179 quotes) → iterative clustering into themes → cross-validation by authors → naming themes → constructing process and content models."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. Authors are at Fraunhofer IESE (applied research institute) and University of Duisburg-Essen, but no grants or sponsors are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Fraunhofer IESE, Kaiserslautern and University of Duisburg-Essen. No products are being evaluated that would create a conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The study does not evaluate any pre-trained model on a benchmark; it is an interview study about developer practices."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for this interview study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite the study involving human interview participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table I reports participant roles, company sizes, application domains, requirements types used, and interaction modes for all 18 participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B states participants 'were selected based on their experience with code models in the implementation phase' and describes deliberate inclusion of diverse roles and domains."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative interview study, not an experimental study with treatment conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study; blinding is not applicable to qualitative interviews."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "All 18 scheduled participants completed their interviews; no attrition is evident (all 18 are listed in Table I and referenced in findings)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Qualitative interview study with no computational method whose cost would be relevant."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments performed."
       }
     }
   },
   "claims": [
     {
       "claim": "Requirements artifacts as traditionally documented are too abstract for direct input into LLMs for code generation.",
       "evidence": "Multiple participants (P04, P10) describe failed attempts at using requirements directly. P10: 'I've never actually had anything ready to use come out of it.' Confirmed by P01, P04-P11, P13, P15, P17 (Section IV.A).",
       "supported": "strong"
     },
     {
       "claim": "Practitioners manually decompose requirements into programming tasks before using them as LLM input.",
       "evidence": "12 of 18 participants explicitly confirmed this decomposition process (P01, P04-P11, P13, P15, P17). P06 describes structuring the program, deciding on functions and database connections before involving ChatGPT (Section IV.A).",
       "supported": "strong"
     },
     {
       "claim": "Three distinct interaction patterns exist: incremental code generation, manual coding with intelligent auto-completion, and extensive code generation.",
       "evidence": "Section IV.A identifies these patterns with specific participant attributions: incremental (most participants), auto-completion (P02, P03, P05, P18), extensive (P04, P08, P13, P14).",
       "supported": "moderate"
     },
     {
       "claim": "Design decisions and architectural constraints must be added to prompts for generated code to be usable in existing codebases.",
       "evidence": "Content model (Section IV.B, Fig. 2) supported by P04, P05, P06, P08, P11 describing the need for infrastructure, language/library, interface, and unit test context in prompts.",
       "supported": "strong"
     },
     {
       "claim": "The decomposition of requirements into programming tasks is not new to LLM-assisted development but mirrors traditional implementation thinking.",
       "evidence": "P04, P05, P06 explicitly argue this in Section IV.A. P04: 'I think this is also the classic thought process of how to implement requirements before coding assistants existed.'",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Convenience sampling bias",
       "detail": "Participants were recruited through the authors' professional network (convenience sampling), which may not represent the broader developer population. The authors acknowledge this in threats to validity but the sample skews toward practitioners connected to a German applied research institute."
     },
     {
       "flag": "Small sample for theory construction",
       "detail": "18 interviews across 14 companies. While theoretical saturation is claimed, the sample may not capture practices in large-scale agentic AI development or non-European contexts."
     },
     {
       "flag": "No ethics approval mentioned",
       "detail": "The study interviews human participants about their professional practices but does not mention IRB or ethics board approval, which is standard for human subjects research."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper — foundational LLM-for-code model evaluated on HumanEval benchmark."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM used in code generation research and practice."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["D. Guo", "Q. Zhu", "D. Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Major open-source code model evaluated for code intelligence tasks."
     },
     {
       "title": "Navigating the complexity of generative AI adoption in software engineering",
       "authors": ["D. Russo"],
       "year": 2024,
       "relevance": "Studies GenAI adoption in SE workflows; directly relevant to understanding LLM integration challenges."
     },
     {
       "title": "Grounded copilot: How programmers interact with code-generating models",
       "authors": ["S. Barke", "M. B. James", "N. Polikarpova"],
       "year": 2023,
       "relevance": "Identifies developer interaction patterns with Copilot; this paper extends those patterns to requirements context."
     },
     {
       "title": "Expectation vs. experience: Evaluating the usability of code generation tools powered by large language models",
       "authors": ["P. Vaithilingam", "T. Zhang", "E. L. Glassman"],
       "year": 2022,
       "relevance": "User study evaluating Copilot usability; found it doesn't reduce task completion time but serves as useful starting point."
     },
     {
       "title": "Productivity assessment of neural code completion",
       "authors": ["A. Ziegler", "E. Kalliamvakou", "X. A. Li"],
       "year": 2022,
       "relevance": "GitHub Copilot productivity assessment — key paper in LLM-assisted developer productivity research."
     },
     {
       "title": "ClarifYGPT: A framework for enhancing LLM-based code generation via requirements clarification",
       "authors": ["F. Mu", "L. Shi", "S. Wang"],
       "year": 2024,
       "relevance": "Attempts to improve LLM code generation through requirements clarification; critiqued in this paper for using programming tasks rather than real requirements."
     },
     {
       "title": "Swe-bench: Can language models resolve real-world github issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLMs on real-world SE tasks."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu"],
       "year": 2024,
       "relevance": "Comprehensive SLR finding requirements engineering and software design highly underrepresented in LLM-for-SE studies."
     }
   ]
 }