ai-research-survey

scan.json (31784B)

---

 {
   "paper": {
     "title": "A Retrieval-Augmented Generation Approach to Extracting Algorithmic Logic from Neural Networks",
     "authors": [
       "Waleed Khalid",
       "Dmitry Ignatov",
       "Radu Timofte"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2512.04329",
     "doi": "10.48550/arXiv.2512.04329"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "NN-RAG extracts dependency-closed, executable PyTorch modules from heterogeneous repositories via scope-aware dependency resolution and three-stage validation (AST parse, bytecode compilation, sandboxed execution). Applied to 19 repositories, the pipeline targeted 1,289 blocks and validated 941 (73.0% pass rate), with 82% qualifying as structurally unique. Multi-level deduplication shows NN-RAG supplies 72.46% of all unique architectures in the LEMUR dataset. The framework also surfaced a CIFAR-10 model achieving 92.81% accuracy, the best in LEMUR at time of writing.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repositories are provided: NN-RAG at https://github.com/ABrain-One/nn-rag (ref [10]) and NN-DUP at https://github.com/ABrain-One/nn-dup (ref [9]). Working URLs are cited in the references."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The LEMUR dataset [19] is publicly available. The paper states 'We release the nn-dup configuration and logs alongside the code to make this tally reproducible and auditable.' The extracted blocks can be regenerated from the open-source tool against public repositories."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions dependencies (LibCST, SQLite, concurrent.futures) and CLI usage but provides no requirements.txt, Dockerfile, conda environment file, or dedicated environment setup section with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "CLI examples are provided in Listing 1 (e.g., 'python3 -m ab.rag --block BertLayer') and a programmatic example in Listing 2, but there are no step-by-step instructions for reproducing the main experimental results (1,289 → 941 validated blocks, CIFAR-10 accuracy). The repository configuration JSON and indexing procedure are described but not as a runnable reproduction protocol."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states: 'Using a Wilson binomial interval at 95% confidence, the pass rate lies in 70.5%–75.4%.' This is a proper confidence interval for the validation pass rate."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. The CIFAR-10 model comparison (Fig 7) and extraction success comparisons are presented as raw numbers without any significance testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Percentage values are reported (73.0% pass rate, 72.46% of unique set from NN-RAG, 92.81% CIFAR-10 accuracy) but no formal effect sizes (Cohen's d, relative improvement with context) are provided for key comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 19 repositories and 1,289 blocks is not justified with power analysis or sample size reasoning. The paper says repositories were 'carefully curated' but does not explain why 19 is sufficient or how the configuration was determined."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported for any results. Extraction time is reported as '~2.5 s per block' (a point estimate). CIFAR-10 accuracy is a single number (92.81%) with no variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The CIFAR-10 evaluation compares against the top 10 models in the LEMUR dataset (Fig 7). The deduplication analysis compares NN-RAG extractions against pre-existing LEMUR content (Table 4: 72.46% NN-RAG vs 27.54% other)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The LEMUR dataset (ref [19], 2025) is contemporary. The comparison is against other models in the same current dataset snapshot."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes performing ablations in Section 6: 'we ablate each factor in isolation (remove SE; replace anti-aliased downsampling with stride-2 convolutions; disable stochastic depth; and swap the augmentation recipe)' but does not present numerical ablation results anywhere in the paper text. The methodology is described without showing the actual accuracy deltas."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 reports extraction success rate, validation pass rate, dependency resolution accuracy, code quality score, average extraction time, and cache hit rate. The deduplication analysis adds uniqueness metrics (Table 4). CIFAR-10 uses accuracy."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states: 'Manual inspection of 50 random blocks showed: (i) formatting preserved in 98%, (ii) dependency completeness in 94%, and (iii) PEP 8 import organization.' This constitutes human evaluation of the system's outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The CIFAR-10 result is reported on the 'standard CIFAR-10 test split' (Fig 7 caption), which is a standard held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 breaks down target blocks by category (attention ~180, convolutional ~220, transformer ~150, etc.). Fig 2 shows per-repository extraction counts. Table 4 provides deduplication statistics by type. Failure causes are categorized (C++/CUDA ops 25%, circular dependencies 20%, etc.)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 categorizes the 348 failures: 'external C++/CUDA ops (~87; 25%), complex/circular dependencies (~70; 20%), dynamic metaprogramming (~52; 15%), and repo-specific utilities/configuration (~139; 40%).' Section 6 further discusses failure mechanisms related to dynamic imports."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The 27% failure rate is prominently reported. Specific failure categories and their causes are documented. The paper acknowledges 'native operators that require toolchains and GPU runtimes are out of scope, and dynamic factories remain partially opaque to static closure.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 1,289 candidates, 941 validated (73.0%), 80%+ structurally unique, and ~72% of LEMUR unique architectures are all supported by Table 3, Table 4, and Section 5 results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'recombining underused practices can yield a robust deep model' and that specific ingredients (SE attention, anti-aliased downsampling, stochastic depth, augmentation recipe) drive the CIFAR-10 improvement. Ablation methodology is described but actual numerical results are not presented, making these causal claims inadequately supported."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states: 'our evaluation therefore prioritizes vision blocks and tasks, using a vision-centric repository configuration while keeping all mechanisms of NN-RAG unchanged.' The scope is bounded to 19 PyTorch repositories and vision-centric evaluation. The abstract focuses on 'vision code.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations for the results. For example, the 72.46% share of unique architectures in LEMUR could be influenced by how the deduplication thresholds (τ=0.90, κ=0.95) were chosen, but this is deferred to a supplement. The CIFAR-10 result could reflect training recipe rather than architectural novelty, but ablation results are not shown."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures executability (AST parse, compile, sandboxed run) as a proxy for 'reusability' but does not discuss the gap. A module that compiles and runs is not necessarily reusable in practice—it may require additional context, documentation, or API compatibility. The paper frames validated blocks as 'a vetted palette for ablations and compositional design' without acknowledging the proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Listing 2 references 'ab.gpt.TuneNNGen_8B' suggesting an 8B-parameter language model for synthesis, but the exact model, version, and provider are not specified. The main extraction pipeline does not use LLMs, but the synthesis integration lacks version details."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The core NN-RAG extraction pipeline is a static analysis and code generation system, not LLM-based prompting. The optional NN-GPT synthesis uses prompts but is not the main contribution evaluated."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Deduplication thresholds are reported (τ=0.90, κ=0.95). Listing 2 shows lr=0.01, momentum=0.9 for the CIFAR-10 example. However, the full training recipe for the winning CIFAR-10 model (epochs, batch size, weight decay, augmentation parameters) is not comprehensively reported—only described qualitatively (RandAugment, mixup/CutMix)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. NN-RAG is a deterministic code analysis and extraction pipeline, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The 7-phase extraction pipeline is documented in detail (Section 4.3, Figure 3): block discovery → repository cloning → LibCST parsing → symbol discovery → dependency resolution → code generation → validation. Deduplication steps are documented in Section 6 with counts at each stage (Table 4: exact 104,804, lexical 8,939, structural 320)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 contains a dedicated limitations paragraph: 'Limitations reflect deliberate sandbox constraints: native operators that require toolchains and GPU runtimes are out of scope, and dynamic factories remain partially opaque to static closure.' Section 7 expands on these limitations with planned mitigations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are identified: native C++/CUDA operators cannot be validated in the sandbox, dynamic import mechanisms (string-to-class registries, plugin loaders) are partially opaque to static analysis, and the repository selection is vision-centric. These are specific to this system rather than generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states scope boundaries: limited to PyTorch codebases, vision-centric repository selection, code-only clones (no weights), and sandbox constraints excluding native operators. Section 5 states the evaluation 'prioritizes vision blocks and tasks.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The extracted block files are intentionally not redistributed due to licensing ('Our extractor never redistributes third-party source files'). While the tool is released to regenerate extractions, the actual raw extracted blocks from the evaluation are not directly available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection process is well-documented: 19 specific repositories are listed (Table 1), the 7-phase extraction pipeline is described in detail (Section 4.3), and the automated block discovery criteria are specified (inherits nn.Module, implements forward(), non-abstract)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The 19 repositories are described as 'carefully curated' representing 'state of the art in computer vision, natural language processing, and graph neural networks,' but the selection criteria and curation process are not fully documented. No explanation for why these specific 19 were chosen over alternatives."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: 1,289 targets discovered → 1,289 extracted (100%) → 941 validated (73.0%) → 348 failures categorized by cause. The deduplication pipeline is also documented (Table 4): 10,483 → 1,064 after exact/lexical/structural dedup."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section is present in the paper. The work is from the University of Würzburg but no grants or sponsors are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: 'Computer Vision Lab, CAIDAS & IFI, University of Würzburg, Germany.' The authors evaluate their own systems (NN-RAG and NN-DUP), and their institutional affiliation is clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence since funding is not disclosed. Absence of funding disclosure prevents verification."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a code extraction pipeline, not a pre-trained model's knowledge capabilities on a benchmark. The CIFAR-10 model is trained from scratch on extracted architectural blocks, not a pre-trained LLM being tested."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. NN-RAG is a deterministic code extraction system, and the CIFAR-10 model is trained from scratch."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Benchmark contamination in the LLM sense is not applicable. The system extracts code via static analysis, and the CIFAR-10 model is trained from scratch rather than tested for pre-existing knowledge."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The research involves automated code extraction from public repositories."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes public code repositories."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 reports: 'Average extraction time was ~2.5 s per block; cache hits on repeat runs reached ~95%.' Section 6 adds: 'Cold-start indexing of 19 repositories completes in ~5–10 minutes; subsequent runs under the missing policy complete in < 30 seconds.' Wall-clock time is reported for the pipeline."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated—no GPU hours for CIFAR-10 training, no hardware specifications, no total API spend. Only per-block extraction time and cold-start indexing time are mentioned."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds for any experiment. The CIFAR-10 accuracy (92.81%) appears to be from a single run with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated for the CIFAR-10 evaluation. The extraction pipeline is deterministic (no randomness), but the training evaluation does not state run count."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Deduplication thresholds are stated as defaults (τ=0.90, κ=0.95) with sensitivity deferred to supplement. For the CIFAR-10 model training, no hyperparameter search budget is reported. No mention of how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 'winning' CIFAR-10 model was surfaced through the pipeline. The paper states this was 'unintentional,' but does not explain the selection process or whether this was the best of many NN-RAG configurations tested."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No multiple statistical comparisons are performed. The paper does not run multiple significance tests that would require correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (NN-RAG) and its contribution to LEMUR (which they also develop). This self-comparison bias is not acknowledged or addressed. The deduplication pipeline (NN-DUP) that determines NN-RAG's contribution share is also their own tool."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of compute budget vs performance across methods. The CIFAR-10 comparison in Fig 7 ranks by accuracy without reporting compute cost for each model."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether CIFAR-10 accuracy is a valid measure for the broader claim of 'algorithmic discovery.' LEMUR's construct validity as a benchmark for neural architecture quality is not examined."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. NN-RAG is a deterministic extraction pipeline, and the CIFAR-10 evaluation is standard model training."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The extracted blocks come from repositories that may have been optimized for CIFAR-10 or similar benchmarks over time, but this potential source of bias is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the extraction process introduces information leakage. The pipeline selects modules from repositories that already contain optimized components for well-known benchmarks."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of non-independence between extracted modules. Many blocks come from the same repositories (e.g., transformers, timm) and share design patterns, but this structural non-independence is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. The deduplication pipeline addresses duplicate architectures but not leakage of benchmark-specific information."
       }
     }
   },
   "claims": [
     {
       "claim": "NN-RAG extracted 1,289 candidate blocks and validated 941 as executable (73.0% pass rate) across 19 repositories.",
       "evidence": "Table 3 and Section 5 report 100% extraction coverage and 941/1,289 validated blocks. Wilson 95% CI: 70.5%–75.4%.",
       "supported": "strong"
     },
     {
       "claim": "Over 80% of extracted blocks are structurally unique (81.93% uniqueness rate).",
       "evidence": "Table 4 and Section 7 state 771 of 941 validated modules qualify as unique after three-level deduplication (exact, lexical MinHash/LSH, AST fingerprint).",
       "supported": "moderate"
     },
     {
       "claim": "NN-RAG supplies approximately 72% of all novel network structures in the LEMUR dataset.",
       "evidence": "Table 4 shows 771 NN-RAG extractions out of 1,064 unique records (72.46%), after deduplication from 10,483 total LEMUR records.",
       "supported": "moderate"
     },
     {
       "claim": "The best CIFAR-10 model in LEMUR was assembled using NN-RAG, achieving 92.81% accuracy.",
       "evidence": "Figure 7 shows the NN-RAG-assembled model (rag-6d58587b76d7e03be409f7e7289d4a58) at 92.81% accuracy on CIFAR-10 test split, topping the LEMUR leaderboard.",
       "supported": "weak"
     },
     {
       "claim": "NN-RAG enables cross-repository migration of architectural patterns, automatically identifying reusable modules in one project and regenerating them dependency-complete in another context.",
       "evidence": "Described in the abstract and methodology (Section 4) as a capability of the system, but no experiment directly demonstrates cross-repo migration with before/after evaluation.",
       "supported": "weak"
     },
     {
       "claim": "The winning model's performance stems from convergent underused ingredients: pre-activation residual backbone, SE attention, anti-aliased downsampling, stochastic depth, and modern augmentation recipe.",
       "evidence": "Section 6 describes ablation methodology (removing each factor in isolation) but does not present numerical ablation results in the paper.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Ablation described but results not shown",
       "detail": "Section 6 describes a systematic ablation (removing SE, replacing anti-aliased downsampling, disabling stochastic depth, swapping augmentation) but the actual numerical results of these ablations are never presented. The paper claims ablations 'pinpoint the primary driver(s)' without showing the data."
     },
     {
       "flag": "Self-evaluation of own system and dataset",
       "detail": "The authors evaluate their own tool (NN-RAG) using their own deduplication tool (NN-DUP) on a dataset they contribute to (LEMUR). There is no independent evaluation or external validation. The claim that NN-RAG supplies 72% of unique architectures depends entirely on their own deduplication pipeline and threshold choices."
     },
     {
       "flag": "Single-run benchmark result without variance",
       "detail": "The CIFAR-10 accuracy of 92.81% (claimed as best in LEMUR) is reported as a single number without error bars, multiple runs, or seed sensitivity analysis. This makes it impossible to assess whether the result is stable or a lucky run."
     },
     {
       "flag": "Sensitivity to deduplication thresholds not shown",
       "detail": "The key claim that NN-RAG contributes 72% of unique architectures depends on deduplication thresholds (τ=0.90, κ=0.95). Sensitivity analysis is deferred to a supplement ('See supplement for sensitivity to τ, κ') rather than presented in the main paper."
     },
     {
       "flag": "Approximate counts throughout",
       "detail": "Table 2 uses approximate counts (~180, ~220, ~150, etc.) rather than exact numbers for the distribution of target blocks. Several metrics use tilde notation (~2.5s, ~95%, ~5-10 minutes) without reporting actual distributions."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM-based software engineering capabilities on real GitHub issues."
     },
     {
       "title": "SWE-bench+: Enhanced Coding Benchmark for LLMs",
       "authors": ["Rehab Aleithan"],
       "year": 2024,
       "arxiv_id": "2410.06992",
       "relevance": "Enhanced version of SWE-bench addressing limitations in the original benchmark evaluation methodology."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["Jiawei Yang"],
       "year": 2024,
       "relevance": "Agentic framework for automated software engineering with explicit interfaces for editing, testing, and navigating code."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "authors": ["Fengji Zhang", "Bei Chen", "Yue Zhang", "Jacky Keung", "Jin Liu", "Daoguang Zan", "Yi Mao", "Jian-Guang Lou", "Weizhu Chen"],
       "year": 2023,
       "arxiv_id": "2303.12570",
       "relevance": "Integrates iterative retrieval with code LLMs for repository-level completion, closely related to RAG-based code systems."
     },
     {
       "title": "CodeR: Issue Resolving with Multi-Agent and Task Graphs",
       "authors": ["Dong Chen"],
       "year": 2024,
       "arxiv_id": "2406.01304",
       "relevance": "Multi-agent system for automated software engineering task resolution using task graphs."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus", "Fabio Petroni", "Vladimir Karpukhin", "Naman Goyal", "Heinrich Küttler", "Mike Lewis", "Wen-tau Yih", "Tim Rocktäschel", "Sebastian Riedel", "Douwe Kiela"],
       "year": 2020,
       "relevance": "Foundational RAG paper establishing the retrieval-augmented generation paradigm used in NN-RAG."
     },
     {
       "title": "A Survey on Hallucination in Large Language Models: Principles, Taxonomy, Challenges, and Open Questions",
       "authors": ["Lei Huang", "Weijiang Yu", "Weitao Ma"],
       "year": 2023,
       "arxiv_id": "2311.05232",
       "relevance": "Survey on LLM hallucination relevant to understanding why retrieval-first approaches reduce generation risks."
     },
     {
       "title": "Do Users Write More Insecure Code with AI Assistants?",
       "authors": ["N. Perry"],
       "year": 2023,
       "relevance": "Studies security risks of AI coding assistance, motivating NN-RAG's validator-gated approach over unconstrained generation."
     },
     {
       "title": "Improving Reproducibility in Machine Learning Research",
       "authors": ["Joelle Pineau"],
       "year": 2021,
       "relevance": "Establishes community reproducibility guidelines that NN-RAG's design claims to follow."
     },
     {
       "title": "Deep Code Search",
       "authors": ["Xiaodong Gu", "Hongyu Zhang", "Sunghun Kim"],
       "year": 2018,
       "relevance": "Deep learning approach to code search using joint embeddings, a predecessor to retrieval-augmented code systems."
     },
     {
       "title": "Large-Scale Near-Deduplication Behind BigCode",
       "authors": ["BigCode"],
       "year": 2023,
       "relevance": "Documents deduplication methodology for large code datasets, relevant to understanding code uniqueness measurement."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "NN-RAG is a released tool that ML practitioners could use to discover and extract reusable PyTorch modules, though it targets a specialized workflow (neural architecture mining)."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that retrieval-first approaches can surface a CIFAR-10 winner is mildly surprising, but the overall approach (code mining + validation) is conventional."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the paper focuses on code extraction and reuse."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with existing work; positioned as complementary to existing tools."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is released on GitHub with CLI examples (python3 -m ab.rag --block ResNet), though not a one-click demo."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "University of Würzburg Computer Vision Lab is not widely known outside the CV research community."
     }
   }
 }