ai-research-survey

scan.json (21333B)

---

 {
   "paper": {
     "title": "Ask-EDA: A Design Assistant Empowered by LLM, Hybrid RAG and Abbreviation De-hallucination",
     "authors": ["Luyao Shi", "Michael Kazda", "Bradley Sears", "Nick Shropshire", "Ruchir Puri"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2406.06575"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL is provided in the paper. The system references internal IBM tools and data, and no GitHub or Zenodo link is given."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The three evaluation datasets (q2a-100, cmds-100, abbr-100) are described as domain-specific to IBM chip design and are not publicly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using LangChain, ChromaDB, and specific models but does not provide a requirements.txt, Dockerfile, or detailed library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a conceptual level but without runnable scripts or detailed setup guides."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Figures 3 and 4 are presented as bar charts with point estimates only. No confidence intervals, error bars, or uncertainty measures are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims hybrid RAG yields 'significantly superior results' and ADH 'significantly boosts performance' but provides no statistical significance tests (no p-values, t-tests, or any formal test)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract reports percentage improvements with context: 'over a 40% improvement in Recall on the q2a-100 dataset and over a 60% improvement on the cmds-100 dataset compared to not using RAG' and 'over a 70% enhancement in Recall on the abbr-100 dataset.' Results are also shown in bar charts with baseline comparisons, providing enough context to gauge magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Each evaluation dataset contains 100 examples. No justification is given for why 100 was chosen, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single runs with no indication of multiple experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares hybrid RAG against no RAG, sparse-only RAG (BM25), and dense-only RAG (sentence transformer) as baselines (Section III.C, Figures 3 and 4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are component ablations (dense-only, sparse-only, no RAG) rather than comparisons against other contemporary RAG systems or design assistant tools. ChipNeMo is mentioned as prior art but not benchmarked against."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation structure functions as an ablation: comparing hybrid RAG vs. dense-only, sparse-only, and no RAG, plus testing with and without the ADH component (Figures 3 and 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: ROUGE-Lsum F1 and ROUGE-Lsum Recall (Section III.C)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of Ask-EDA's outputs is reported. The paper mentions collecting feedback via Slack GUI but explicitly states 'the feedback data are not used in the evaluation study in this paper' (Section II.E). Given the system is a design assistant for engineers, human evaluation of response quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "It is unclear whether the evaluation datasets were used for any tuning or development decisions. No explicit separation of dev and test splits is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across three distinct datasets (q2a-100, cmds-100, abbr-100), each targeting a different task aspect, and across two models (Figures 3 and 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases: Llama2-13b-chat's lower F1 despite comparable Recall ('it is plausible that Llama2-13b-chat struggles to effectively extract the final answer'), and that neither model achieves 1.0 recall on abbr-100 despite abbreviation knowledge being in the prompt, attributing this to 'intrinsic limitations in LLMs' (Section III.C)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Llama2-13b-chat does not show improvement from hybrid over sparse/dense in terms of F1 on q2a-100, and that neither model achieves perfect recall on abbreviations even with ADH (Section III.C)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims of 40%, 60%, and 70% improvements are directionally supported by the bar charts in Figures 3 and 4, though exact numbers are read from charts rather than stated in tables."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims that hybrid RAG and ADH improve performance. These are supported by controlled ablation experiments where components are added/removed while holding other factors constant (Figures 3 and 4). This single-variable manipulation constitutes adequate causal design for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusion claim Ask-EDA 'can effectively respond to design-related inquiries' broadly, but evaluation is on three proprietary IBM chip design datasets with only two 13B-parameter models. No bounding to the specific design domain, document corpus, or model scale is provided."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. For example, the paper does not consider whether improvements could be due to the specific characteristics of the evaluation datasets rather than the methods, or whether ROUGE is an appropriate metric for measuring answer quality in this domain."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'Granite-13b-chat-v2.1' and 'Llama2-13b-chat' (Section III.B). The sentence transformer is also specified: 'all-MiniLM-L6-v2'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The system prompt is provided verbatim in Section III.B: 'You are a helpful AI language model. Your primary function is to assist users in answering questions...' The ADH prompt format is also specified in Section II.C."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported in Section III.B: context length 8192, max new tokens 4096, chunk size 2048, chunk overlap 256, ndense=nsparse=nhybrid=3, RRF constant k=60."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The RAG pipeline scaffolding is described in detail: document ingestion (Section II.B.1), retrieval with hybrid search and RRF (Section II.B.2), abbreviation matching and injection (Section II.C), LLM generation (Section II.D), and Slack interface (Section II.E). Diagrams are provided in Figures 1 and 2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II.A describes document sources (400MB, 10,200 command pages, 5,000 parameters, 30 slack channels, 18,000 Q&A pairs). Section II.B.1 describes ingestion using LangChain document loaders with supported formats. Section III.A describes how each evaluation dataset was derived."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The Discussion (Section IV) briefly mentions future directions but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. Issues such as the proprietary nature of datasets, small evaluation sizes, use of only ROUGE as a metric, and lack of human evaluation are not acknowledged."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what its results do not show, or what settings/domains/models are excluded from its claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation datasets and raw results are not made publicly available. The data is described as proprietary IBM chip design content."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III.A describes how each dataset was collected: q2a-100 from a stack-overflow type system with expert-marked answers, cmds-100 from tool manual pages with command synopses turned into questions, abbr-100 randomly sampled from the 249-term abbreviation dictionary."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation. The datasets are curated from existing internal documents and tools."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline from document sources through ingestion (LangChain loaders, chunking, embedding, BM25 indexing) to retrieval and generation is documented in Sections II.A-II.D with accompanying diagrams (Figures 1 and 2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. All authors are from IBM, and the Acknowledgment section thanks IBM management for 'support and guidance' but does not explicitly disclose funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: IBM Research (Shi, Puri) and IBM Infrastructure (Kazda, Sears, Shropshire). The paper evaluates IBM's own Granite model alongside Llama2."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "IBM employees built and evaluated the system using IBM's own Granite model. IBM has a commercial interest in demonstrating the effectiveness of both the Ask-EDA system and the Granite model. The funder (IBM) is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Given that the work evaluates IBM's own products, this omission is notable."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states for Granite-13b-chat-v2.1 that 'There is no overlap between the LLM training data and our ingested data' (Section III.B), but does not state any training data cutoff date for either model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "For Granite-13b-chat-v2.1, the paper explicitly states 'There is no overlap between the LLM training data and our ingested data' (Section III.B). No such statement is made for Llama2-13b-chat, but the data is proprietary IBM content unlikely to be in public training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation datasets are curated from proprietary IBM internal systems (internal Q&A platform, internal tool manuals, internal abbreviation dictionary), making benchmark contamination unlikely. The paper notes 'There is no overlap between the LLM training data and our ingested data' for Granite."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the evaluation study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed per query is reported, despite the system making multiple retrieval and LLM calls per query."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No information about hardware used, GPU hours, or computational resources is provided."
       }
     }
   },
   "claims": [
     {
       "claim": "Hybrid RAG offers over a 40% improvement in Recall on q2a-100 compared to not using RAG",
       "evidence": "Figure 3 shows bar chart comparisons of hybrid vs. no-RAG configurations on q2a-100 for both models (Section III.C).",
       "supported": "moderate"
     },
     {
       "claim": "Hybrid RAG offers over a 60% improvement in Recall on cmds-100 compared to not using RAG",
       "evidence": "Figure 3 shows that no-RAG achieves 0 Recall on cmds-100 while hybrid RAG achieves non-zero scores. Exact values are read from bar charts (Section III.C).",
       "supported": "moderate"
     },
     {
       "claim": "ADH yields over a 70% enhancement in Recall on abbr-100",
       "evidence": "Figure 4 shows comparison of hybrid RAG with and without ADH on abbr-100 (Section III.C). Results shown only as bar charts without exact numbers or statistical tests.",
       "supported": "moderate"
     },
     {
       "claim": "Hybrid search RAG outperforms both sparse-only and dense-only RAG",
       "evidence": "Figure 3 shows hybrid consistently achieving highest or comparable scores across datasets and models. For Granite on q2a-100, hybrid is best. On cmds-100, hybrid outperforms. For Llama2, the pattern holds for Recall but not F1 on q2a-100 (Section III.C).",
       "supported": "moderate"
     },
     {
       "claim": "Adding ADH does not adversely affect performance on cmds-100 and q2a-100",
       "evidence": "Figure 4 shows results for cmds-100 and q2a-100 with and without ADH, demonstrating comparable performance (Section III.C).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "Ask-EDA combines hybrid RAG (dense + sparse retrieval with reciprocal rank fusion) and an abbreviation de-hallucination component to assist EDA engineers. Hybrid RAG substantially improves retrieval performance over no RAG, sparse-only, and dense-only approaches on three proprietary IBM evaluation datasets. The ADH component significantly improves abbreviation resolution without degrading performance on other tasks. However, neither model achieves perfect abbreviation recall even when correct information is present in the prompt, suggesting intrinsic LLM limitations.",
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "All authors are IBM employees evaluating IBM's Granite model alongside an open-source baseline. Granite is shown to outperform Llama2 on F1 across all conditions. No competing interests statement is provided."
     },
     {
       "flag": "No statistical tests despite 'significant' claims",
       "detail": "The paper repeatedly uses the word 'significantly' (e.g., 'significantly superior results,' 'significantly boosts performance') without any statistical significance testing. All claims are based on visual comparison of bar charts."
     },
     {
       "flag": "No error bars or variance",
       "detail": "All results appear to be from single runs with no error bars, confidence intervals, or multi-run variance reported."
     },
     {
       "flag": "Proprietary non-reproducible evaluation",
       "detail": "All three evaluation datasets are IBM-internal and proprietary. No code is released. The results cannot be independently verified or reproduced."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity section despite multiple methodological concerns (small datasets, single metric family, no human evaluation, proprietary data)."
     },
     {
       "flag": "ROUGE as sole metric family",
       "detail": "Only ROUGE-Lsum variants (F1 and Recall) are used. ROUGE measures lexical overlap and may not capture semantic correctness, particularly problematic for design Q&A where paraphrased but correct answers would score low."
     }
   ],
   "cited_papers": [
     {
       "title": "GPT-4 Technical Report",
       "authors": ["J. Achiam", "S. Adler", "S. Agarwal"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM used in many agentic AI and code generation evaluations."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["P. Lewis", "E. Perez", "A. Piktus"],
       "year": 2020,
       "relevance": "Introduces RAG, a core technique for grounding LLM outputs used widely in agentic AI systems."
     },
     {
       "title": "ChipNeMo: Domain-Adapted LLMs for Chip Design",
       "authors": ["M. Liu", "T.-D. Ene", "R. Kirby"],
       "year": 2023,
       "arxiv_id": "2311.00176",
       "relevance": "Directly comparable domain-adapted LLM system for chip design assistance, evaluating LLM productivity tools for engineers."
     },
     {
       "title": "Granite Code Models: A Family of Open Foundation Models for Code Intelligence",
       "authors": ["M. Mishra", "M. Stallone", "G. Zhang"],
       "year": 2024,
       "arxiv_id": "2405.04324",
       "relevance": "Describes the Granite model family used in this evaluation, relevant to LLM code generation capabilities."
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback",
       "authors": ["L. Ouyang", "J. Wu", "X. Jiang"],
       "year": 2022,
       "relevance": "Introduces RLHF for instruction-following, foundational technique for LLM alignment and safety."
     }
   ]
 }