ai-research-survey

scan.json (32043B)

---

 {
   "paper": {
     "title": "Natural Language Outlines for Code: Literate Programming in the LLM Era",
     "authors": [
       "Kensen Shi",
       "Deniz Altınbüken",
       "Saswat Anand",
       "Mihai Christodorescu",
       "Katja Grünwedel",
       "Alexa Koenings",
       "Sai Naidu",
       "Anurag Pathak",
       "Marc Rasi",
       "Fredde Ribeiro",
       "Brandon Ruffin",
       "Siddhant Sanyam",
       "Maxim Tabachnyk",
       "Sara Toth",
       "Roy Tu",
       "Tobias Welp",
       "Pengcheng Yin",
       "Manzil Zaheer",
       "Satish Chandra",
       "Charles Sutton"
     ],
     "year": 2025,
     "venue": "FSE Companion '25 (33rd ACM International Conference on the Foundations of Software Engineering)",
     "arxiv_id": "2408.04820",
     "doi": "10.1145/3696630.3728541"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "Modern LLMs (especially Gemini 1.5 Pro) can generate high-quality NL outlines for code functions, with 60% rated excellent and 80% completely correct by professional developers on 30 real Python functions. In an Android security case study, 30 professional reverse engineers found LLM-generated outlines very or extremely helpful (26/30), with 83.8% of outlines rated completely correct and LLM suspicion scores correlating more highly with experienced REs (r=0.85) than less experienced groups. A Virtual CL Split prototype for code review was considered useful for 58% of sufficiently complex change lists, and was launched to ~7000 beta users at Google.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code, repository URL, or archive is provided anywhere in the paper. Prototype features (Finish Changes, Virtual CL Split) are described but not released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The dataset of 30 Python functions, 80 Android security functions, and 73 CLs used in evaluations are not released. The paper notes functions come from 'real projects' and 'real Android apps' but provides no download link or repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications (requirements.txt, Dockerfile, library versions) are provided. The paper only names the LLMs used without specifying the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While Appendix E provides example prompts, there are no instructions for replicating the full experimental pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., '60% of outlines were rated as having excellent overall quality', '83.8% said the outline was completely correct') without any confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Comparisons between models and techniques (e.g., 'Gemini 1.5 Flash was the next best LLM and the others were noticeably worse') are made by comparing raw percentages without any hypothesis tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pearson correlation coefficients are reported for LLM vs. RE suspicion scores (r=0.85, r=0.80, r=0.78, r=0.96 in Section 6.1). Percentage breakdowns per model/technique in Fig. 5 and per experience group in Fig. 7 provide magnitude context for comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification is provided for any study. The choice of 30 functions, 6 evaluators (Section 5), 80 functions, 30 REs (Section 6.1), or 73 CLs, 4 engineers (Section 6.2) is not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across evaluators or experimental conditions. All results are single-point aggregates (percentages, counts). Greedy decoding is used so LLM outputs are deterministic, but no inter-rater variability is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 compares 5 LLMs (Gemini 1.5 Pro/Flash, Gemini 1.0 Pro/Ultra, DeepSeek-Coder-Instruct 33B) and 2 generation techniques (Interleaved Generation, Line Number Infilling). Section 6.1 compares 3 prediction styles (suspicion score, summary, NL outline)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Gemini 1.5 Pro and Flash (2024 models) are used alongside Gemini 1.0 Pro/Ultra and DeepSeek-Coder-Instruct 33B, representing a mix of contemporary and recent models at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. While different generation techniques and models are compared, individual components of the approach (e.g., few-shot examples, prompt instructions, number of examples) are not systematically removed or varied to measure their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 evaluates 5 dimensions: overall quality, helpfulness, correctness, amount of detail, and style/fluency. Section 6.1 evaluates accuracy (suspicion scores), correctness, helpfulness, and detail. Table 1 reports parsing errors and number of outline statements."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 uses 6 professional developers to rate outline quality. Section 6.1 surveys 30 professional reverse engineers on LLM prediction accuracy and helpfulness. Section 6.2 has 4 engineers evaluate virtual CL splits."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The 30 evaluation functions in Section 5 are separate from the 8 handwritten few-shot examples used in prompts. The evaluation set was curated independently from the prompt examples."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 5 breaks down quality ratings by LLM and generation technique. Fig. 7 breaks down helpfulness by prediction type and RE experience group. Table 1 provides per-model parsing error rates. Section 6.1 separates results for suspicious vs. benign functions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 notes '3 of 30 outlines were bad quality, 2 were not helpful, and only 1 was incorrect.' Section 6.1 discusses 4 false negatives and 2 false positives with detailed analysis. Fig. 16 shows a low-quality outline example with commentary. App. C discusses mistakes in Finish Changes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Gemini 1.0 Pro 'often includes too much detail' (Section 5), Line Number Infilling 'led to slightly worse predictions' (Section 5), weaker LLMs have more formatting errors (Table 1), and in the code review study only 58% of splits for complex CLs were useful (Section 6.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: '60% of outlines were rated by professionals as excellent overall (90% were acceptable or better)' matches Fig. 5 results; '26 of 30 professional reverse engineers said that outlines are very or extremely helpful' matches Fig. 7; '83.8% of our generated outlines were rated as completely correct' matches Section 6.1 results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to frame most claims as survey findings ('60% were rated excellent', '26/30 said helpful') rather than strong causal claims. The comparative claims (Model A vs Model B) are adequately supported by the controlled comparison design. The broader use-case claims ('can accelerate understanding') are explicitly framed as proposals, not empirical findings."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Natural Language Outlines for Code' and abstract claims about 'code' generally are not bounded to the tested setting. Section 5 uses only Python (acknowledged in limitations: 'only in Python which is our expertise'). Section 6.1 uses only Java/Android. The paper proposes use cases across 'all of the different software development surfaces' from evidence in limited settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the positive survey results are discussed. Possible confounds include: novelty effect (participants may rate any new tool positively), demand characteristics (evaluators in Section 5 include 5 paper authors evaluating their own functions), Hawthorne effect in the RE study, and selection bias in function curation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures perceived helpfulness and quality via surveys but frames results in terms of 'accelerating understanding and navigation' (abstract) and 'improving developer efficiency' (Section 2) without prominently distinguishing the proxy (survey ratings) from the outcome (actual productivity/understanding improvement). Section 7 briefly acknowledges 'Further research is needed to measure the perceived and actual benefits' but this distinction is not salient in the main claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are named as 'Gemini 1.0 Pro and Ultra', 'Gemini 1.5 Flash and Pro', and 'DeepSeek-Coder-Instruct 33B' (Section 5). No snapshot dates, API versions, or specific version identifiers are provided. Marketing names like 'Gemini 1.5 Pro' without a snapshot date do not count as specified versions per the schema."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides full example prompts for both Interleaved Generation and Line Number Infilling techniques, including system instructions, few-shot examples (2-3 shown, 8 used in practice), and user query templates. Actual few-shot content is shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states 'all with greedy decoding,' which specifies the sampling strategy (temperature=0, deterministic). The paper uses 'a fixed set of 8 handwritten few-shot examples.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is direct few-shot prompting of LLMs without tool use, retry logic, or multi-step agentic workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 documents function curation: 30 Python functions from 21 projects, 10-90 LOC (median 46.5), outline-like comments removed from 7 functions, other comments preserved in 10 functions. Section 6.1 describes the 80-function dataset: 40 suspicious (manually curated), 40 benign (randomly sampled from 8 popular apps), 24-99 LOC, controlled for code length and style distributions."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Discussion) includes a dedicated 'Limitations' subsection that discusses specific limitations of the three studies."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: 'In Section 5 we used 6 survey participants, and in Section 6.2 we used 4 participants from one team, so those findings may be specific to those participants.' Also: 'the 30 dataset functions were not sampled uniformly' with 'bias toward research projects' and 'Our choice of functions can introduce bias.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states specific scope boundaries: evaluation only in Python ('only in Python which is our expertise'), biased function selection ('many functions are boilerplate, very short or simple, or otherwise less interesting'), small evaluator pools, and 'A full evaluation of each use case is beyond the scope of this paper' (Section 3)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. Individual survey responses, function datasets, generated outlines, and RE ratings are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes the function curation process (6 professionals, 21 real projects, variety criteria). Section 6.1 describes the Android dataset (40 suspicious from prior RE flags, 40 benign from popular apps, length/style matching). Section 6.2 describes CL selection (73 randomly selected from one team's recent submissions, >10 lines changed)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "While participants are characterized (professional SEs/researchers in Section 5, professional REs in Section 6.1, engineers from one team in Section 6.2), the recruitment process, channels used, and potential recruitment bias are not described. Footnote 1 notes '5 of the dataset contributors are also authors' but does not discuss how the other evaluator was recruited."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: function curation → comment removal → LLM generation with specified prompts → programmatic parsing (with error classification detailed in App. F) → survey presentation (shuffled, unlabeled) → rating collection. For the Android study: function collection → LLM prediction (3 styles in one query with 7 few-shot examples) → RE survey with balanced assignments."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is explicitly disclosed. The acknowledgments section thanks individuals but does not mention grants, corporate funding, or funding agencies. The work was presumably funded by Google given all authors' affiliation, but this is not stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed with 'Google' affiliation on the first page. The connection between the authors' employer and the primary models evaluated (Gemini, a Google product) is clear from the affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google funds this research (all authors are Google employees) and has direct commercial interest in positive findings about Gemini's capabilities for developer tools. Google's Gemini models are the primary focus of evaluation, and the paper's positive results support Google's product narrative."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Google employees evaluating Google's Gemini product is a clear potential conflict that is not explicitly acknowledged beyond listing affiliations."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 5 LLMs used (Gemini 1.0 Pro/Ultra, Gemini 1.5 Flash/Pro, DeepSeek-Coder). The paper cannot assess whether evaluation functions were in training data without this information."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 30 evaluation functions or 80 Android functions could have been in the models' training data. Footnote 2 notes 'The LLMs in our experiments were not trained on the removed comments' but this implicitly acknowledges the code itself may be in training data without addressing it."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation functions come from 'real projects' (some from authors' prior work) which could be publicly available and in training data. Section 5 notes 'custom dependencies... not in the prompt or training data' but this refers to dependency code, not the functions themselves. No systematic contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for any of the three human evaluation studies."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite three studies involving human participants (6 developers, 30 REs, 4 engineers)."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Demographics are minimal. Section 5: '6 professional software engineers and researchers.' Section 6.1: '30 professional REs' split into '3 groups of 10 by their amount of experience' — no further detail on experience levels, years, or other demographics. Section 6.2: 'four software engineers from the same team.' No gender, geographic, age, or detailed experience information is provided."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No explicit inclusion or exclusion criteria are stated for any study's participants. The only characterization is professional role (SEs, researchers, REs) without formal eligibility criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The studies are survey/rating studies, not randomized experiments comparing treatment vs. control conditions. Participants all see the same types of outputs to rate."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 / App. G describes single-blinding: 'presenting each function's 10 outlines in shuffled order to the function's contributor' without labeling which LLM or technique produced each outline. This prevents evaluator bias toward specific models."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition or dropout information is reported for any study. It is unclear whether all invited participants completed their evaluations."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No systematic cost or latency reporting is provided. Relative speedup ('Line Number Infilling provides a 4× to 5× speedup,' Section 4) and one latency estimate ('about 5 seconds' for Finish Changes with Flash, App. C) are mentioned in passing but no API costs, tokens consumed, or per-example costs are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The experiments involve generating 300 outlines (Section 5), predictions for 80 functions (Section 6.1), and splits for 73+ CLs (Section 6.2), but GPU hours, total API costs, and hardware used are not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "60% of NL outlines generated by Gemini 1.5 Pro with Interleaved Generation were rated as having excellent overall quality (90% acceptable or better).",
       "evidence": "Section 5 and Fig. 5: Quality survey of 30 Python functions rated by their curators (6 professional developers) across 5 dimensions. Results shown in bar charts per model/technique.",
       "supported": "moderate"
     },
     {
       "claim": "80% of outlines from Gemini 1.5 Pro with Interleaved Generation were rated completely correct.",
       "evidence": "Section 5 and Fig. 5: Same survey, correctness dimension. 17% were 'mostly correct' and only 1 of 30 was incorrect.",
       "supported": "moderate"
     },
     {
       "claim": "83.8% of NL outlines for suspicious Android functions were rated as completely correct by professional reverse engineers.",
       "evidence": "Section 6.1: 30 professional REs rated LLM predictions. Per-function surveys collected with each function rated by one RE from each of 3 experience groups.",
       "supported": "strong"
     },
     {
       "claim": "26 of 30 professional reverse engineers said NL outlines are very or extremely helpful for malware detection.",
       "evidence": "Section 6.1 and Fig. 7: Final survey about predictions overall. REs rated suspicion scores, summaries, and outlines for helpfulness, broken down by experience group.",
       "supported": "strong"
     },
     {
       "claim": "LLM suspicion scores correlate more highly with the most experienced REs (r=0.85) than other RE groups correlate with them (r=0.80, r=0.78).",
       "evidence": "Section 6.1: Pearson correlation between LLM scores and each RE experience group's scores on the 80-function dataset. LLM scores also achieve r=0.96 against the average of all RE scores.",
       "supported": "moderate"
     },
     {
       "claim": "Line Number Infilling provides 4-5x speedup over Interleaved Generation for outline generation.",
       "evidence": "Section 4: Described as an 'observed' speedup because Line Number Infilling does not spend output tokens repeating the code. No formal timing measurements reported.",
       "supported": "weak"
     },
     {
       "claim": "Virtual CL Split was useful for 58% of sufficiently complex change lists during code review.",
       "evidence": "Section 6.2: Preliminary study with 4 engineers on 73 CLs. 24 of 73 CLs were deemed complex enough; 14 of those 24 splits were considered useful.",
       "supported": "weak"
     },
     {
       "claim": "The LLM achieves 5% false negative rate and 0% false positive rate on Android malware suspicion scoring.",
       "evidence": "Section 6.1: Of 40 suspicious functions, 4 had predicted score 0 (false negatives), but further analysis shows 2 were correct predictions, leaving 2 actual false negatives. Of 40 benign functions, 2 had nonzero scores but REs unanimously agreed these were accurate.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "All 20 authors are Google employees evaluating Google's Gemini models as the primary system. Gemini 1.5 Pro is found to be the best model across most dimensions. The one non-Google baseline (DeepSeek-Coder 33B) is substantially smaller and older, making the comparison favorable to Gemini."
     },
     {
       "flag": "Evaluators are paper authors",
       "detail": "In Section 5, the 6 evaluators who curated the 30 functions also rated the outlines for those functions. Footnote 1 confirms '5 of the dataset contributors are also authors of this paper.' This creates a clear conflict — authors have incentive to rate outlines favorably."
     },
     {
       "flag": "Small and non-representative samples",
       "detail": "Section 5 uses only 6 evaluators and 30 curated Python functions (biased toward research projects, only Python). Section 6.2 uses only 4 engineers from one team. These are too small for the broad claims about NL outlines as a general developer assistance paradigm."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "Claims that one model or technique outperforms another are made by comparing raw percentages without any significance tests, confidence intervals, or uncertainty quantification. With 30 functions and 6 evaluators, differences could easily be due to chance."
     },
     {
       "flag": "Baseline fairness concern",
       "detail": "DeepSeek-Coder-Instruct 33B (a 33B parameter open-source model) is compared against Gemini 1.5 Pro (a much larger model) without acknowledging the substantial model size disparity. This makes the comparison inherently favorable to the Google models."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["Angela Fan", "Beliz Gokkaya", "Mark Harman", "Mitya Lyubarskiy", "Shubho Sengupta", "Shin Yoo", "Jie M Zhang"],
       "year": 2023,
       "relevance": "Survey of LLMs applied to software engineering tasks, directly relevant to understanding the landscape of AI-assisted development."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu", "Zhou Yang", "Kailong Wang", "Li Li", "Xiapu Luo", "David Lo", "John Grundy", "Haoyu Wang"],
       "year": 2023,
       "arxiv_id": "2308.10620",
       "relevance": "Systematic review of LLMs in SE, providing context for how NL outlines fit within the broader LLM-for-SE research space."
     },
     {
       "title": "In-IDE Human-AI Experience in the Era of Large Language Models; A Literature Review",
       "authors": ["Agnia Sergeyuk", "Sergey Titov", "Maliheh Izadi"],
       "year": 2024,
       "arxiv_id": "2401.10739",
       "relevance": "Literature review of human-AI interaction in IDEs, directly relevant to NL outlines as an IDE integration concept."
     },
     {
       "title": "Unifying the perspectives of NLP and software engineering: A survey on language models for code",
       "authors": ["Ziyin Zhang", "Chaoyu Chen", "Bingchang Liu", "Cong Liao", "Zi Gong", "Hang Yu", "Jianguo Li", "Rui Wang"],
       "year": 2023,
       "arxiv_id": "2311.07989",
       "relevance": "Survey on language models for code generation and understanding, covering the foundational capabilities that enable NL outline generation."
     },
     {
       "title": "Source Code Summarization in the Era of Large Language Models",
       "authors": ["Weisong Sun", "Yun Miao", "Yuekang Li", "Hongyu Zhang", "Chunrong Fang", "Yi Liu", "Gelei Deng", "Yang Liu", "Zhenyu Chen"],
       "year": 2024,
       "arxiv_id": "2407.07959",
       "relevance": "Survey of LLM-based code summarization, the most closely related capability to NL outline generation."
     },
     {
       "title": "A comparative analysis of large language models for code documentation generation",
       "authors": ["Shubhang Shekhar Dvivedi", "Vyshnav Vijay", "Sai Leela Rahul Pujari", "Shoumik Lodh", "Dhruv Kumar"],
       "year": 2024,
       "relevance": "Evaluates LLMs for code documentation generation, a closely related application to NL outline generation."
     },
     {
       "title": "Using an LLM to help with code understanding",
       "authors": ["Daye Nam", "Andrew Macvean", "Vincent Hellendoorn", "Bogdan Vasilescu", "Brad Myers"],
       "year": 2024,
       "relevance": "Studies how LLMs can help developers understand code, directly relevant to the code understanding use case of NL outlines."
     },
     {
       "title": "Expectations, outcomes, and challenges of modern code review",
       "authors": ["Alberto Bacchelli", "Christian Bird"],
       "year": 2013,
       "relevance": "Foundational study on code review practices, providing context for the Virtual CL Split application of NL outlines."
     },
     {
       "title": "Modern code review: a case study at Google",
       "authors": ["Caitlin Sadowski", "Emma Söderberg", "Luke Church", "Michal Sipko", "Alberto Bacchelli"],
       "year": 2018,
       "relevance": "Case study of code review at Google, directly relevant to the Virtual CL Split feature evaluated at Google."
     },
     {
       "title": "\"What it wants me to say\": Bridging the abstraction gap between end-user programmers and code-generating large language models",
       "authors": ["Michael Xieyang Liu", "Advait Sarkar", "Carina Negreanu", "Benjamin Zorn", "Jack Williams", "Neil Toronto", "Andrew D Gordon"],
       "year": 2023,
       "relevance": "Studies the abstraction gap in code generation from NL, relevant to the bidirectional NL-code sync proposed for NL outlines."
     },
     {
       "title": "Enhancing LLM-Based Coding Tools through Native Integration of IDE-Derived Static Context",
       "authors": ["Yichen Li", "Yun Peng", "Yintong Huo", "Michael R Lyu"],
       "year": 2024,
       "arxiv_id": "2402.03630",
       "relevance": "Proposes enhancing LLM coding tools with IDE context, relevant to the RAG-based improvements discussed for NL outlines."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming – The Rise of Code Intelligence",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang", "Zhenda Xie", "Kai Dong", "Wentao Zhang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Describes DeepSeek-Coder, one of the models evaluated in the NL outline generation experiments."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "NL outlines are a practical concept for developer tooling with working prototypes demonstrated, but no tool is released for practitioners to use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Extends literate programming and code summarization ideas with LLMs — evolutionary rather than contrarian, though the bidirectional sync concept is novel."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the Android security application is about detecting malware, not creating it."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims or conflicts with prior work; a constructive proposal paper."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released; only mockups, screenshots, and prototype descriptions are provided."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "From Google with evaluation of Gemini models; well-known lab and product but not as attention-grabbing as OpenAI/ChatGPT papers."
     }
   }
 }