ai-research-survey

scan.json (24712B)

---

 {
   "paper": {
     "title": "Assessing the Answerability of Queries in Retrieval-Augmented Code Generation",
     "authors": ["Geonmin Kim", "Jaeyeon Kim", "Hancheol Park", "Wooksu Shin", "Tae-Ho Kim"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2411.05547",
     "doi": "10.48550/arXiv.2411.05547"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub URL (https://github.com/Nota-NetsPresso/RaCGEval) in Section 1: 'RaCGEval benchmark dataset and pre-trained answerability assessment models are publicly available.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The RaCGEval benchmark dataset is publicly released at the GitHub URL. The paper also uses publicly available datasets (CoNaLa, TorchDataEval, BeatNumEval, MonkeyEval from Zan et al., 2022)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided in the paper. The paper mentions using QLoRA and specific models but does not list library versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While the benchmark and models are released, the paper does not include a README with commands to run or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports only point estimates of accuracy (e.g., 46.7%) with no confidence intervals or error bars. Figure 4 and Figure 5 also show point estimates without uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., fine-tuning improves over zero-shot, ICL improves accuracy) but provides no statistical significance tests. Differences between models and methods are compared by raw numbers only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw accuracy percentages (e.g., 31.2%, 33.0%, 46.7%) but does not provide effect sizes with baseline context in a standardized format. While absolute differences can be inferred, no formal effect size measures are reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 1,016 total samples (Table 1) but no justification is given for why this size is adequate. No power analysis or discussion of whether the sample supports the statistical claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experimental results. It is unclear whether experiments were run multiple times or represent single runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes multiple baselines: zero-shot inference on three LLMs (gpt-3.5, llama3, gemma), fine-tuned versions, and in-context learning variants. Table 2 and Figure 4 compare these approaches."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines use contemporary models including gpt-3.5-turbo-0613, llama3-instruct-8b, and gemma-1.1-7b-it, all of which were recent at the time of writing (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 provides ablation-like analysis: Section 5.1 examines the effect of in-context learning (comparing zero-shot vs. ICL, with and without fine-tuning), and Section 5.2 analyzes the trade-off between coverage and precision. These show which components matter."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses accuracy for the classification task (Table 2) and pass@k (k=10) for code generation quality (Figure 5). Coverage vs. precision trade-off is also analyzed."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 describes human annotation by four programming experts, with an additional two experts for inter-annotator agreement validation using Fleiss' Kappa (0.7408). The humans evaluate the benchmark labels, which is part of the system's ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly describe a train/dev/test split for the RaCGEval benchmark. The training data (Section 3.2.2: 2192 answerable, 2191 unanswerable, 2185 partially answerable from CoNaLa) is separate from the benchmark, but there is no mention of a held-out dev set used for hyperparameter tuning vs. the test benchmark."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides per-domain breakdowns (NetsPressoEval, TorchDataEval, BeatNumEval, MonkeyEval) for zero-shot, ICL, and fine-tuned settings across multiple LLM backbones."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific failure cases: unanswerable types not covered, the gap between verification accuracy and pass@k ('the LLM can generate code without the gold API documents, using its prior knowledge'), and the limitations of domain adaptation via random ICL examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that zero-shot performance is near chance level (~33%) for all models (Table 2), that 'the highest verification accuracy does not always achieve the best pass@k' (Limitations), and discusses where fine-tuning alone is insufficient without domain adaptation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the task is 'very challenging' with baseline models at '46.7%' accuracy. Table 2 confirms this (gemma fine-tuned: 46.7%). The abstract also mentions methods to 'significantly improve performance,' supported by Section 5 (ICL improvements shown in Figure 4)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claims are that fine-tuning and ICL improve accuracy. These are supported by controlled comparisons: same models with/without fine-tuning (Table 2), same models with/without ICL (Figure 4). The ablation design is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and title frame the work broadly as 'Retrieval-Augmented Code Generation' but the benchmark only covers four specific libraries (NetsPresso, TorchData, BeatNum, Monkey), three of which are modified/private versions. The paper does not bound its generalization claims to these specific domains."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for its results. For example, the poor zero-shot performance could be due to prompt design, model unfamiliarity with modified API names (BeatNum, Monkey), or the three-class nature of the task. These alternatives are not considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 specifies exact model versions: 'gemma-1.1-7b-it', 'llama3-instruct-8b', and 'gpt-3.5-turbo-0613'. These are specific enough to identify the exact models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The prompt template is provided in full in Table 4 (Appendix B), and the few-shot examples are provided in Tables 5-8 (Appendix D). The actual prompt text is given, not just a natural language description."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using QLoRA for fine-tuning but does not report key hyperparameters such as learning rate, number of epochs, LoRA rank, temperature for inference, or other sampling parameters. Section 4.1 states the method but omits these details."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a straightforward classification pipeline: prompt -> LLM -> prediction token."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the API documentation sources. Section 2.3 describes in detail the three methods for generating partially answerable and unanswerable samples, including similarity thresholds (tau_r = 0.8, tau_i = 0.1) and the CodeT5+ encoder used. Section 3.2 describes training data construction."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "There is a dedicated 'Limitations' section after the Conclusion that discusses three specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section raises specific threats: (1) RaCGEval may not cover all real-world unanswerable types, (2) random ICL examples may not be optimal for domain adaptation, (3) verification accuracy and pass@k are not perfectly correlated because LLMs can use prior knowledge. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings are excluded. While the Limitations section discusses potential improvements, it does not delineate clear scope boundaries (e.g., 'our results apply only to these four libraries' or 'we did not test larger models')."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The RaCGEval benchmark dataset is publicly released at the GitHub URL (https://github.com/Nota-NetsPresso/RaCGEval), including pre-trained models."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the four API documentation sources in detail. Sections 2.3 and 2.4 describe how samples were generated and annotated, including the similarity thresholds and methods used."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.4 mentions 'four programming experts' and 'additional two experts' for annotation but does not describe how these annotators were recruited, their qualifications, or potential selection bias. The paper only states they are 'programming experts' without further characterization."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: API documentation sources (Section 2.2), answerable sample construction, three methods for generating unanswerable/partially answerable samples with similarity thresholds (Section 2.3), annotation process with inter-annotator agreement (Section 2.4), and final dataset statistics (Table 1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work was supported by Artificial intelligence industrial convergence cluster development project funded by the Ministry of Science and ICT(MSIT, Korea) & Gwangju Metropolitan City.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Nota Inc. The paper header clearly states 'Nota Inc.' with author emails at nota.ai."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is a Korean government project (Ministry of Science and ICT & Gwangju Metropolitan City), which appears independent of the specific outcomes of this benchmark study."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Notably, all authors are from Nota Inc., the company that develops NetsPresso, which is one of the four libraries in the benchmark. This potential conflict is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses gpt-3.5-turbo-0613, llama3-instruct-8b, and gemma-1.1-7b-it on the benchmark but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 explicitly addresses this: 'the benchmark dataset includes APIs that most language models are unlikely to have encountered during training.' They use private (NetsPressoEval) or modified (BeatNumEval, MonkeyEval with rephrased keywords) API documentation specifically to avoid overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is designed to mitigate contamination: NetsPressoEval uses private API docs, BeatNumEval and MonkeyEval use 'keywords and structure rephrased' versions of NumPy/Pandas. Section 2.2 explains this was done 'to ensure a fair assessment that excludes prior knowledge of each LLM.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The human involvement is limited to expert annotation of the benchmark dataset, not a human subjects study. No participant-level data is collected about the annotators."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are programming experts performing a professional task (labeling data), not human subjects in a research study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The annotators are described only as 'programming experts' in their professional capacity."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Annotator selection is a professional staffing decision, not participant recruitment."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. All 6 annotators completed their annotation tasks."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper motivates the task by mentioning computational cost savings ('reduce computational costs for LLM service providers') but does not report actual inference costs, API costs, tokens consumed, or latency for any of the models evaluated."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No information is provided about GPU hours, training time, total API spend, or hardware used for the experiments. The paper mentions using QLoRA to reduce VRAM but does not quantify the actual compute budget."
       }
     }
   },
   "claims": [
     {
       "claim": "The answerability assessment task in RaCG remains very challenging, with baseline models exhibiting a low performance of 46.7%.",
       "evidence": "Table 2 shows the best model (gemma fine-tuned) achieves 46.7% accuracy. Zero-shot baselines range from 31.2% to 36.9%, near the 33.3% chance level for 3-way classification.",
       "supported": "strong"
     },
     {
       "claim": "In-context learning substantially improves accuracy compared to zero-shot inference, suggesting domain adaptation is crucial.",
       "evidence": "Figure 4 shows ICL-3w1s improves accuracy across all backbones and domains compared to zero-shot, both with and without fine-tuning. The paper states 'the RaCG benchmark prioritizes LLM learning domain information (i.e., by ICL) over learning task information (i.e., by fine-tuning).'",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning LLMs improves answerability assessment over zero-shot inference.",
       "evidence": "Table 2: llama3 improves from 33.0% (zero-shot) to 36.5% (fine-tuned), gemma from 36.9% to 46.7%.",
       "supported": "moderate"
     },
     {
       "claim": "The RaCGEval benchmark annotation is reliable, with substantial inter-annotator agreement (Fleiss' Kappa = 0.7408).",
       "evidence": "Section 2.4 reports Fleiss' Kappa of 0.7408 across three sets of labels from 6 total annotators, interpreted as 'substantial agreement' per Landis & Koch (1977).",
       "supported": "strong"
     },
     {
       "claim": "Introducing answerability assessment creates a controllable trade-off between coverage and code generation precision.",
       "evidence": "Figure 5 shows pass@k vs. coverage curves for various verification models, demonstrating that reducing coverage (accepting fewer queries as answerable) increases the precision of generated code.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces RaCGEval, a benchmark for evaluating whether LLMs can determine if a user query is answerable given retrieved API descriptions in retrieval-augmented code generation. Zero-shot LLM performance is near chance level (~33%) on the 3-way classification task, while fine-tuning reaches 46.7%, indicating the task is challenging. In-context learning with domain-specific examples provides larger improvements than task-specific fine-tuning alone, suggesting domain adaptation is the primary bottleneck. The paper also demonstrates a practical coverage-precision trade-off: filtering queries through an answerability model before code generation improves code quality at the cost of coverage.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All authors are from Nota Inc., which develops NetsPresso. NetsPressoEval is one of the four benchmark subsets, and the paper promotes NetsPresso API documentation as a use case. This conflict of interest is not explicitly acknowledged."
     },
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results (Table 2, Figures 4-5) are reported as point estimates with no confidence intervals, error bars, or indication of variance across runs. It is unclear if experiments were run multiple times."
     },
     {
       "flag": "Missing hyperparameter details",
       "detail": "Key training hyperparameters (learning rate, epochs, LoRA rank, inference temperature) are not reported despite using QLoRA fine-tuning, making reproduction difficult."
     },
     {
       "flag": "Obfuscated benchmark names may confuse difficulty assessment",
       "detail": "BeatNumEval and MonkeyEval are modified versions of NumPy and Pandas with rephrased keywords. While this mitigates contamination, it also means LLMs face an artificially difficult task due to unfamiliar naming rather than genuine reasoning difficulty. This confound is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and pass@k metric, foundational for LLM code generation evaluation."
     },
     {
       "title": "QLoRA: Efficient Finetuning of Quantized LLMs",
       "authors": ["Tim Dettmers", "Artidoro Pagnoni", "Ari Holtzman", "Luke Zettlemoyer"],
       "year": 2023,
       "relevance": "Parameter-efficient fine-tuning method used as the training approach in this paper's experiments."
     },
     {
       "title": "RAGAS: Automated Evaluation of Retrieval Augmented Generation",
       "authors": ["Shahul Es", "Jithin James", "Luis Espinosa Anke", "Steven Schockaert"],
       "year": 2024,
       "relevance": "Method for evaluating RAG hallucinations, directly compared in the related work as an alternative approach."
     },
     {
       "title": "SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models",
       "authors": ["Potsawee Manakul", "Adian Liusie", "Mark Gales"],
       "year": 2023,
       "relevance": "Hallucination detection method using response consistency, discussed as a related approach for detecting unreliable LLM outputs."
     },
     {
       "title": "LLatrieval: LLM-Verified Retrieval for Verifiable Generation",
       "authors": ["Xiaonan Li", "Changtai Zhu", "Linyang Li"],
       "year": 2024,
       "relevance": "Provides the prompt template used for zero-shot answerability assessment in this paper's experiments."
     },
     {
       "title": "CodeHalu: Code Hallucinations in LLMs Driven by Execution-Based Verification",
       "authors": ["Yuchen Tian", "Weixiang Yan", "Qian Yang"],
       "year": 2024,
       "arxiv_id": "2405.00253",
       "relevance": "Benchmark for evaluating code hallucinations in LLMs, directly relevant to the code generation reliability topic."
     },
     {
       "title": "CodeT5+: Open Code Large Language Models for Code Understanding and Generation",
       "authors": ["Yue Wang", "Hung Le", "Akhilesh Gotmare"],
       "year": 2023,
       "relevance": "Code LLM whose encoder is used for computing semantic similarity between queries and APIs in the benchmark construction."
     },
     {
       "title": "When Language Model Meets Private Library",
       "authors": ["Daoguang Zan", "Bei Chen", "Zeqi Lin"],
       "year": 2022,
       "relevance": "Source of three of the four benchmark datasets (TorchDataEval, BeatNumEval, MonkeyEval) used in RaCGEval."
     },
     {
       "title": "Private-Library-Oriented Code Generation with Large Language Models",
       "authors": ["Daoguang Zan"],
       "year": 2023,
       "arxiv_id": "2307.15370",
       "relevance": "Follow-up work on private library code generation, directly relevant to the RaCG task setting."
     },
     {
       "title": "HaluEval: A Large-Scale Hallucination Evaluation Benchmark for Large Language Models",
       "authors": ["Junyi Li", "Xiaoxue Cheng", "Xin Zhao"],
       "year": 2023,
       "relevance": "Large-scale hallucination benchmark for LLMs, related to the broader goal of detecting unreliable LLM outputs."
     },
     {
       "title": "Retrieval Augmentation Reduces Hallucination in Conversation",
       "authors": ["Kurt Shuster", "Spencer Poff", "Moya Chen"],
       "year": 2021,
       "relevance": "Foundational work on RAG showing it reduces hallucination, motivating the RaCG setting studied in this paper."
     }
   ]
 }