ai-research-survey

scan.json (24684B)

---

 {
   "paper": {
     "title": "Towards AI-Native Software Engineering (SE 3.0): A Vision and a Challenge Roadmap",
     "authors": [
       "Ahmed E. Hassan",
       "Gustavo A. Oliva",
       "Dayi Lin",
       "Boyuan Chen",
       "Zhen Ming (Jack) Jiang"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2410.06107"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or repository link is provided in the paper. The paper describes a technology stack (Compiler.next, Runtime.next, etc.) but does not release any implementation artifacts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The paper is a vision paper that draws on surveys of literature, discussions with industry leaders, and the authors' practical experience, but none of these inputs are made publicly available."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision/position paper with no experiments to reproduce. Environment specifications are not applicable."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision/position paper with no experiments. There are no results to reproduce."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision/position paper that does not present its own quantitative experimental results. No statistical analysis is performed."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No comparative experiments are conducted in this paper, so significance tests are not applicable."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted in this paper. Some numbers are mentioned from companion papers (e.g., '30% latency improvement', '50% fewer requests routed'), but these are results from other papers, not from this paper's own analysis."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with sample sizes are conducted in this paper."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted in this paper. Variance reporting is not applicable."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision paper that does not conduct experiments. SE 2.0 is discussed as a conceptual contrast to SE 3.0, but no empirical baseline comparison is made."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted, so baseline recency is not applicable."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system is evaluated in this paper; ablation studies are not applicable."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted in this paper."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs are evaluated in this paper. Human evaluation is not applicable to a vision paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted in this paper."
       },
       "per_category_breakdown": {
         "applies": false,
         "answer": false,
         "justification": "No experimental results are presented that could be broken down by category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 critically discusses the limitations and failure cases of SE 2.0, including cognitive overload (Section 2.2.1), inefficient model training (Section 2.2.2), and suboptimal code quality/additive bias (Section 2.2.3). Section 2.3 discusses limitations of autonomous software engineers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges negative aspects: SE 2.0 limitations, SWE-bench intrinsic limitations (Section 2.3), the fragility of prompts (Section 4.5), and the general acknowledgment that SE 3.0 challenges remain unsolved. Section 4.6 also lists open questions the authors have not yet developed a vision for."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the paper proposes a vision for SE 3.0 with intent-centric development and outlines key components and challenges. The body of the paper delivers on these claims: Section 3 presents the vision and technology stack, and Section 4 presents challenges. The claims are appropriately framed as visionary rather than empirical."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes several causal claims without adequate empirical backing, such as 'cognitive overload on developers' being caused by SE 2.0 (Section 2.2.1), and that AI-generated code 'contaminates future training data, creating a feedback loop that further degrades model quality' (Section 2.2.3). These are stated as established facts but are supported only by reasoning and selective citations, not by the authors' own controlled studies."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims about the future of software engineering without clearly bounding them. For example, the abstract states SE 3.0 will feature 'AI systems evolving beyond task-driven copilots into intelligent collaborators' without qualifying this as speculative or conditional. The title frames SE 3.0 as a general paradigm shift. While Section 4 does acknowledge challenges, the vision itself is presented in sweeping, unbounded terms."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative visions or explanations for the future of SE. Section 2.3 briefly considers autonomous software engineers as an alternative but dismisses them. No consideration is given to alternative paradigms (e.g., that SE 2.0 may improve sufficiently, or that different organizational models might address the same problems without the proposed technology stack)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used experimentally in this paper. Model names are mentioned in passing (e.g., 'GPT 5.2', 'DeepSeek V3.2') but only as contextual references, not as models the authors run experiments with."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is performed in this paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted in this paper."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used experimentally in this paper. The proposed technology stack describes conceptual scaffolding but does not implement or evaluate it."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states its vision was 'identified based on (i) surveys of academic and gray literature, (ii) in-depth discussions with industrial and academic leaders...' (Section 1) but provides no documentation of how these inputs were collected, filtered, or synthesized. No systematic methodology is described for the literature survey or expert consultations."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section 4 discusses challenges for realizing SE 3.0, but these are open research problems for the proposed vision, not limitations of the paper's own methodology or analysis."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address potential biases in its methodology (e.g., the authors' affiliation with Huawei and their vested interest in the proposed technology stack)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the vision does NOT cover or where its claims should not be applied. Section 4.6 lists open questions 'for which we have not yet developed a thorough vision,' which is a partial acknowledgment, but no explicit scope boundaries are drawn for the overall vision."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper references surveys, discussions, and practical experience as inputs to its vision, but none of this source material is made available. The literature survey results, meeting notes, and customer feedback that informed the vision are not provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 1 lists five sources of input (surveys, discussions, customer meetings, practical experience, industry partners) but provides no detail on how data was collected from any of these. No timeframes, instruments, or sampling methods are described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited for a study. The expert discussions mentioned are informal consultations, not a structured study with participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented. The transformation from the five input sources (Section 1) to the SE 3.0 vision is not described in any systematic way."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The acknowledgments section disclaims that opinions don't reflect Huawei's, but does not state who funded the research."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: three authors are from the Centre for Software Excellence at Huawei Canada, one from Queen's University, and one from York University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three of the five authors are from Huawei Canada, and the paper's proposed technology stack (Compiler.next, Runtime.next, FMArts, FMware) references Huawei-affiliated work extensively. The funder (implicitly Huawei) has a direct commercial interest in the proposed SE 3.0 vision being adopted."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Three authors work at Huawei, and several referenced companion papers [28, 45, 98, 114] are authored by the same group and describe Huawei-related systems."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted in this paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted in this paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted in this paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted in this paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted in this paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted in this paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted in this paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision/position paper. It does not implement or run its own method. Cost reporting is not applicable."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a vision/position paper with no computational experiments of its own."
       }
     }
   },
   "claims": [
     {
       "claim": "SE 2.0 imposes high cognitive overload on developers because the human drives the code creation loop, requiring continuous problem decomposition, prompting, evaluation, and debugging.",
       "evidence": "Section 2.2.1 describes the programming workflow in detail and cites Vaithilingam et al. [97] on debugging 'rabbit holes'. Also cites He et al. [46] on Cursor increasing code complexity.",
       "supported": "moderate"
     },
     {
       "claim": "AI coding assistants exhibit additive bias, leading to bloated codebases that harm long-term maintainability.",
       "evidence": "Section 2.2.3 cites He et al. [46] finding Cursor leads to higher code complexity and static analysis warnings. Also cites Sergeyuk et al. [89] survey on why developers reject AI suggestions.",
       "supported": "moderate"
     },
     {
       "claim": "SE 3.0 will enable intent-centric, conversation-oriented development where AI drives the code creation loop instead of humans.",
       "evidence": "Section 3 describes this vision conceptually (Sections 3.1-3.6) with a proposed technology stack. No empirical evidence is provided for the vision itself.",
       "supported": "unsupported"
     },
     {
       "claim": "Curriculum engineering is more efficient than unstructured internet-scale data for training FMs.",
       "evidence": "Section 3.6 argues for this philosophically, citing the phi family of models [15] and InstructLab [51, 91] as examples, but provides no direct empirical comparison between curriculum-engineered and data-driven training in this paper.",
       "supported": "weak"
     },
     {
       "claim": "A multi-agent ToM-enhanced conversational system significantly improved intent clarification, requirement completeness, and human-AI alignment across 150 diverse scenarios.",
       "evidence": "Section 4.1 references Gallaba et al. [36] as 'complementary work' providing evidence. This is a separate paper, not results from the current paper.",
       "supported": "weak"
     },
     {
       "claim": "Runtime.next's Fusion Runtime shows approximately 30% latency improvement compared to Ray Serve.",
       "evidence": "Section 4.3 states 'Preliminary results show a latency improvement in the order of 30% compared to Ray Serve,' referencing the companion paper [45]. No details or methodology are provided in this paper.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "theoretical"
   ],
   "key_findings": "This paper proposes Software Engineering 3.0 (SE 3.0), an AI-native paradigm shift from the current AI-assisted SE 2.0. The key vision is intent-centric, conversation-oriented development where AI teammates drive the code creation loop rather than human developers. The proposed technology stack includes five components: Teammate.next (personalized AI partners), IDE.next (intent-centric IDEs), Compiler.next (multi-objective code synthesis), Runtime.next (SLA-aware execution), and FM.next (curriculum-engineered models). The paper identifies six major challenges and eight additional open questions that must be addressed to realize this vision.",
   "red_flags": [
     {
       "flag": "Vendor-affiliated vision paper",
       "detail": "Three of five authors are from Huawei Canada's Centre for Software Excellence. The proposed SE 3.0 technology stack heavily references the authors' own companion papers (Compiler.next [28], FMware [45], Runtime.next [114], RAR [98], Watson [85], SPICE [70], conversational development [36]) and Huawei-related systems (FMArts, Fusion Runtime). The vision is substantially a description of Huawei's research agenda framed as a community roadmap."
     },
     {
       "flag": "No empirical validation of core vision",
       "detail": "The central claims about SE 3.0 being superior to SE 2.0 are purely speculative. All empirical results cited are from companion papers (often under review), not from this paper itself. The paper acknowledges 'the SE 3.0 vision can only be truly assessed and validated as a whole once prototypes have been developed for all components of the stack' (Section 5)."
     },
     {
       "flag": "No limitations section",
       "detail": "For a paper making sweeping claims about the future of software engineering, the absence of a limitations section discussing potential failure modes, risks, or conditions under which the vision might not hold is a significant methodological gap."
     },
     {
       "flag": "Unsystematic methodology for vision development",
       "detail": "The paper lists five sources of input (surveys, expert discussions, customer meetings, practical experience, industry partnerships) but provides no systematic methodology for how these were gathered, analyzed, or synthesized into the proposed vision. This makes the conclusions difficult to evaluate or reproduce."
     },
     {
       "flag": "Undisclosed conflicts of interest",
       "detail": "No competing interests statement despite authors being employed by Huawei (which would commercially benefit from adoption of the proposed stack), membership in OPEA alliance with 40+ companies, and multiple self-citations to companion papers under review."
     }
   ],
   "cited_papers": [
     {
       "title": "Does AI-Assisted Coding Deliver? A Difference-in-Differences Study of Cursor's Impact on Software Projects",
       "authors": ["Hao He", "Courtney Miller", "Shyam Agarwal", "Christian Kästner", "Bogdan Vasilescu"],
       "year": 2025,
       "arxiv_id": "2511.04427",
       "relevance": "Empirical study finding that Cursor boosts short-term velocity but increases code complexity and static analysis warnings, directly relevant to AI coding tool productivity evaluation."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Key study on AI coding assistant productivity impact, directly relevant to the survey's assessment of empirical evidence on LLM programming tools."
     },
     {
       "title": "Measuring GitHub Copilot's Impact on Productivity",
       "authors": ["Albert Ziegler", "Eirini Kalliamvakou", "X. Alice Li", "Andrew Rice"],
       "year": 2024,
       "doi": "10.1145/3633453",
       "relevance": "Empirical measurement of Copilot's productivity impact, relevant to evaluating claims about AI coding assistant effectiveness."
     },
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu"],
       "year": 2024,
       "doi": "10.1145/3695988",
       "relevance": "Comprehensive systematic review of LLMs applied to software engineering tasks, directly relevant as a survey of the LLM-SE space."
     },
     {
       "title": "Using AI-based coding assistants in practice: State of affairs, perceptions, and ways forward",
       "authors": ["Agnia Sergeyuk", "Yaroslav Golubev", "Timofey Bryksin", "Iftekhar Ahmed"],
       "year": 2025,
       "doi": "10.1016/j.infsof.2024.107610",
       "relevance": "Survey of ~500 developers on reasons for not using AI coding assistants, providing empirical evidence on developer trust and perception issues."
     },
     {
       "title": "Rethinking Software Engineering in the Foundation Model Era: A Curated Catalogue of Challenges in the Development of Trustworthy FMware",
       "authors": ["Ahmed E. Hassan", "Dayi Lin", "Gopi Krishnan Rajbahadur"],
       "year": 2024,
       "arxiv_id": "2402.15943",
       "relevance": "Companion paper from the same group cataloguing challenges in foundation-model-powered software development."
     },
     {
       "title": "Agentic Software Engineering: Foundational Pillars and a Research Roadmap",
       "authors": ["Ahmed E. Hassan", "Hao Li", "Dayi Lin"],
       "year": 2025,
       "arxiv_id": "2509.06216",
       "relevance": "Related roadmap paper on agentic software engineering, directly relevant to the survey's scope of agentic AI in SE."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "relevance": "Open platform for AI software engineering agents, relevant to the survey's coverage of agentic coding tools."
     },
     {
       "title": "Expectation vs. Experience: Evaluating the Usability of Code Generation Tools Powered by Large Language Models",
       "authors": ["Priyan Vaithilingam", "Tianyi Zhang", "Elena L. Glassman"],
       "year": 2022,
       "doi": "10.1145/3491101.3519665",
       "relevance": "User study evaluating LLM code generation usability, finding developers can get trapped in debugging rabbit holes."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "Approach for routing requests between LLMs, relevant to the survey's coverage of efficient LLM deployment and cost optimization."
     },
     {
       "title": "Hidden technical debt in Machine learning systems",
       "authors": ["D. Sculley", "Gary Holt", "Daniel Golovin"],
       "year": 2015,
       "relevance": "Foundational paper on ML systems engineering challenges, cited by the authors as inspiration for their systems-level view of AI engineering."
     },
     {
       "title": "Compiler.next: A Search-Based Compiler to Power the AI-Native Future of Software Engineering",
       "authors": ["Filipe R. Cogo", "Gustavo A. Oliva", "Ahmed E. Hassan"],
       "year": 2025,
       "arxiv_id": "2510.24799",
       "relevance": "Companion paper presenting the search-based code synthesis component of the proposed SE 3.0 stack, with initial benchmark evaluation."
     }
   ]
 }