ai-research-survey

scan.json (22769B)

---

 {
   "paper": {
     "title": "A Literature Review on AI-Powered Smart Code Base Navigator",
     "authors": [
       "Radhika S K",
       "Rashmitha R",
       "Sanjana N",
       "Shanthala M N",
       "Sukanya G"
     ],
     "year": 2025,
     "venue": "International Journal of Scientific Research in Engineering and Management (IJSREM)",
     "doi": "10.55041/IJSREM52774"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This literature review surveys 20 papers on AI-powered code search and navigation tools, covering transformer-based code generation, retrieval-augmented generation, semantic code search, and code completion. The paper summarizes each work's methodology, strengths, and weaknesses in a tabular format. No original system is built or evaluated despite the abstract implying one is introduced. The review lacks systematic methodology — no search protocol, no quality assessment of sources, and no structured synthesis beyond per-paper summaries.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code, analysis scripts, or repository links are provided. A survey can release search/analysis scripts but this one does not."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset, search corpus, or structured extraction of reviewed papers is released. The summary table is in the paper but not as downloadable data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or tool specifications are provided for reproducing the literature search or analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No instructions are given for reproducing the literature search — no databases searched, no search queries, no date ranges, no inclusion/exclusion criteria."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a narrative literature review with no experiments or statistical analysis."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; the paper only summarizes existing work."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or statistical aggregation are performed in this survey."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted in this narrative literature review."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments or quantitative synthesis are performed."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare itself against prior surveys or reviews on the same topic. No baseline comparison of any kind."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted; this criterion requires experimental baselines."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system or experimental design to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No evaluation metrics are used; this is a narrative review."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate; this is a literature review."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experimental evaluation requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Papers are organized into labeled categories (A through T) covering code generation, RAG, code search, code completion, etc., with a summary table breaking down methodology and remarks per paper."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The summary table includes a 'Cons' column for each reviewed paper, noting limitations such as 'Can produce invalid or insecure code,' 'Resource-heavy,' 'Focused on education; lacks quantitative comparison,' etc."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Each reviewed paper's weaknesses and limitations are reported. For example, the review notes that UNIF outperforms more complex models (Section T), and that LLMs fail to grasp semantics (Section M)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'This initiative introduces an AI-Powered Smart Code Base Navigator, a system designed to facilitate semantic code search, context-aware code completion, and streamlined navigation.' No such system is actually built, implemented, or evaluated in the paper. The body is entirely a literature review of 20 existing papers."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion claims the navigator 'improves code comprehension, offers context-sensitive support, and shortens the time developers need to find and understand pertinent sections of the code.' No causal evidence is provided — no system exists to test these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims about AI 'revolutionizing software development' and 'greatly enhancing software engineering methodologies' based on a review of 20 papers, without bounding these generalizations to the specific tools, languages, or contexts reviewed."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered for the conclusions drawn. The paper does not discuss whether the improvements reported in reviewed papers could be due to confounds, dataset selection, or other factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames reviewed tools as 'boosting developer productivity' and 'enhancing software engineering methodologies' without discussing whether the metrics reported (MRR, BLEU, accuracy) actually capture productivity or engineering quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used by the authors; this is a literature review."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in this literature review."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted in this survey paper."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; this is a literature review."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper selection pipeline is not documented at all. There is no description of which databases were searched, what search queries were used, what time period was covered, or how the 20 papers were selected from a larger pool."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations section. The conclusion mentions 'future opportunities' but does not discuss limitations of the review itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what types of papers were excluded, what time period was covered, or what the review does NOT claim to show."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The full set of search results, screening decisions, or extracted data from reviewed papers is not provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the 20 reviewed papers were found. No databases, search terms, or collection procedures are mentioned."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For a survey, 'recruitment' means paper selection. No description is given of how papers were identified or selected for inclusion."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No pipeline from initial search to final 20 papers is documented. There are no filtering stages, counts at each stage, or criteria described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: one assistant professor and four UG students from the Department of Computer Science and Engineering, JNNCE, Shivamogga, Karnataka, India."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "This appears to be an unfunded undergraduate student project."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature review."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature review."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature review."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature review."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature review."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature review."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this literature review."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no system or method of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computation performed."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA diagram, no structured search protocol, no reproducible search queries, and no systematic methodology is described. Papers appear to have been selected ad-hoc."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of its source papers. It lists pros and cons narratively but has no quality scoring rubric, risk-of-bias assessment, or structured evaluation. All 20 papers are treated as equally credible."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Publication bias is not discussed. The survey does not consider whether its sources are biased toward positive results or whether negative results in code search/navigation are underrepresented."
       }
     }
   },
   "claims": [
     {
       "claim": "The paper introduces an 'AI-Powered Smart Code Base Navigator' that facilitates semantic code search, context-aware code completion, and streamlined navigation within extensive Python codebases.",
       "evidence": "Abstract and Introduction describe the system conceptually, but the paper body (Sections A-T) is entirely a literature review of 20 existing works. No system design, implementation, or evaluation is presented.",
       "supported": "unsupported"
     },
     {
       "claim": "Meta-RAG achieved about 84.7% accuracy at locating the buggy file and 53.0% at the function level on SWE-bench Lite.",
       "evidence": "Reported in Section B from Tawosi et al. [2]. The claim is a direct report from the cited paper, not independently verified.",
       "supported": "moderate"
     },
     {
       "claim": "EVOR achieved roughly 2–4× higher correct code generation rates than recent baselines like Reflexion or DocPrompting.",
       "evidence": "Reported in Section H from Su et al. [8]. The claim is a direct report from the cited paper, not independently verified.",
       "supported": "moderate"
     },
     {
       "claim": "REINFOREST outperforms previous cross-language search tools by up to 44.7% on accuracy metrics.",
       "evidence": "Reported in Section I from Saieva et al. [9]. The claim is a direct report from the cited paper, not independently verified.",
       "supported": "moderate"
     },
     {
       "claim": "The AI-Powered Smart Code Base Navigator 'improves code comprehension, offers context-sensitive support, and shortens the time developers need to find and understand pertinent sections of the code.'",
       "evidence": "Stated in the Conclusion. No system was actually built or evaluated to support these claims.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Phantom system — abstract describes a system that doesn't exist",
       "detail": "The abstract and introduction claim to 'introduce' an 'AI-Powered Smart Code Base Navigator' system, but the paper body is entirely a literature review of 20 existing papers. No system is designed, implemented, or evaluated. The conclusion then claims the system 'improves code comprehension' without any evidence."
     },
     {
       "flag": "Copy-paste error from template",
       "detail": "The Literature Survey section begins with 'In this section, various authors have presented various Emotion detection techniques.' This is clearly pasted from a different paper's template and is unrelated to code navigation."
     },
     {
       "flag": "No systematic review methodology",
       "detail": "The 20 reviewed papers appear hand-picked without any documented selection criteria, search strategy, databases queried, or inclusion/exclusion criteria. The review is entirely ad-hoc."
     },
     {
       "flag": "No quality assessment of sources",
       "detail": "All 20 reviewed papers are treated as equally credible. No quality scoring, risk-of-bias assessment, or critical evaluation of methodology is performed. This launders the signal-to-noise ratio of the sources."
     },
     {
       "flag": "Predatory journal indicators",
       "detail": "Published in IJSREM (ISSN: 2582-3930), which exhibits characteristics of a predatory journal: no rigorous peer review evident (copy-paste error was not caught), rapid publication, and low editorial standards."
     },
     {
       "flag": "Undergraduate student project published as research",
       "detail": "Four of five authors are listed as 'UG Students.' The paper lacks the methodological rigor expected of a published literature review — no structured protocol, no quality assessment, and claims about a system that was never built."
     }
   ],
   "cited_papers": [
     {
       "title": "Meta-RAG on large codebases using code summarization",
       "authors": ["V. Tawosi", "S. Alamir", "X. Liu", "M. Veloso"],
       "year": 2025,
       "arxiv_id": "2508.02611",
       "relevance": "Multi-agent RAG framework for bug localization in large codebases, achieving SOTA on SWE-bench Lite."
     },
     {
       "title": "Position: Intelligent coding systems should write programs with justifications",
       "authors": ["X. Xu", "S. Feng", "Z. Su", "C. Wang", "X. Zhang"],
       "year": 2025,
       "arxiv_id": "2508.06017",
       "relevance": "Proposes neuro-symbolic approach for AI coding assistants to generate verifiable justifications alongside code."
     },
     {
       "title": "Conversational AI as a coding assistant: Understanding programmers' interactions with and expectations from large language models for coding",
       "authors": ["M. Akhoroz", "C. Yildirim"],
       "year": 2025,
       "arxiv_id": "2503.16508",
       "relevance": "Survey of 143 student developers on LLM chatbot usage for coding, documenting accuracy and over-reliance concerns."
     },
     {
       "title": "A deep dive into retrieval-augmented generation for code completion: Experience on WeChat",
       "authors": ["Z. Yang", "C. Wang", "T. Peng", "H. Huang", "Y. Deng", "C. Gao"],
       "year": 2025,
       "arxiv_id": "2507.18515",
       "relevance": "Empirical study of RAG for C++ code completion in a large closed-source industrial codebase."
     },
     {
       "title": "EVOR: Evolving retrieval for code generation",
       "authors": ["H. Su", "S. Jiang", "Y. Lai", "H. Wu", "B. Shi", "C. Liu", "Q. Liu", "T. Yu"],
       "year": 2024,
       "arxiv_id": "2402.12317",
       "relevance": "RAG pipeline that dynamically updates queries and knowledge base for code generation, claiming 2-4x improvement over baselines."
     },
     {
       "title": "A comprehensive survey of AI-driven advancements and techniques in automated program repair and code generation",
       "authors": ["A. Anand", "N. Yadav", "A. Gupta", "S. Bajaj"],
       "year": 2024,
       "arxiv_id": "2411.07586",
       "relevance": "Systematic review of 27 works on AI-assisted bug fixing and code synthesis, covering search-based repair and LLM-based generation."
     },
     {
       "title": "Code Search Is All You Need? Improving Code Suggestions with Code Search",
       "authors": ["J. Chen", "X. Hu", "Z. Li", "C. Gao", "X. Xia", "D. Lo"],
       "year": 2024,
       "relevance": "Demonstrates retrieval-augmented code suggestion with up to 130.8% improvement in code generation BLEU scores."
     },
     {
       "title": "LLMs: Understanding code syntax and semantics for code analysis",
       "authors": ["W. Ma", "W. Wang", "S. Liu", "Y. Liu", "Q. Hu", "L. Li", "Z. Lin", "C. Zhang", "L. Nie"],
       "year": 2024,
       "arxiv_id": "2305.12138",
       "relevance": "Evaluates GPT-4, GPT-3.5, StarCoder, and CodeLlama on syntax vs. semantic understanding, finding LLMs fail at runtime reasoning."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Z. Feng"],
       "year": 2020,
       "relevance": "Foundational bimodal transformer pre-trained on code and natural language, establishing NL-PL pre-training for code tasks."
     },
     {
       "title": "REINFOREST: Reinforcing Semantic Code Similarity for Cross-Lingual Code Search Models",
       "authors": ["A. Saieva", "S. Chakraborty", "G. Kaiser"],
       "year": 2024,
       "arxiv_id": "2305.03843",
       "relevance": "Cross-language code search method incorporating runtime information into static representations, claiming up to 44.7% accuracy improvement."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Describes the concept of an AI-powered code navigator but no system is actually built; a practitioner gets only a reading list."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms conventional wisdom that AI can help with code search; no novel or surprising findings."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns are raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; purely a summary of existing work."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is released — the described system does not exist."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Published in IJSREM by undergraduate students at a regional Indian engineering college; no brand recognition."
     }
   }
 }