ai-research-survey

scan.json (25274B)

---

 {
   "paper": {
     "title": "A Comparative Study of DSL Code Generation: Fine-Tuning vs. Optimized Retrieval Augmentation",
     "authors": ["Nastaran Bassamzadeh", "Chhaya Methani"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2407.02742"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link, Zenodo archive, or any other code repository is mentioned in the paper. No code release is discussed."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a synthetically generated dataset of 67k training samples and 1000 test samples, but no download link or public release is provided. The datasets are proprietary to Microsoft."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Azure AML pipelines and GPT-4 with 16k token limit but provides no requirements.txt, Dockerfile, library versions, or detailed environment setup information."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided. The methodology is described at a high level but lacks sufficient detail for independent reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-4 are reported as single point estimates (delta improvements) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'TST with FD setting performs overall better') based solely on comparing delta values without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage-point improvements over baselines with enough context to understand magnitude (e.g., 'hallucination rate for API names dropping by 6.29 pts' in Section 6, and all table results shown as delta improvements from named baselines)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test set has 1000 samples and the training set has 67k samples, but no justification is given for why these sizes were chosen and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported. It is unclear whether results are from single runs or averaged over multiple runs. No mention of multiple experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a Pre-Trained model with 5 few-shots as baseline in Tables 1-3, and a fine-tuned Codex model as baseline in Table 4. Multiple ablation variants are compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The fine-tuned model is based on Codex, which was deprecated by OpenAI in 2023. GPT-4 is used for RAG but no comparison against other contemporary code generation systems or approaches (e.g., Code Llama fine-tuning, StarCoder) is made. The baselines are internal ablation variants rather than competitive external systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper presents a systematic ablation study across Sections 5.1-5.3, varying: number of few-shots (5 vs 20), retrieval model (Pre-trained vs TST), and grounding type (no FD, FD, SFD, FD+SFD). Tables 1-4 show results for each ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: Average Similarity (LCSS-based), %Unparsed flows, %made-up API names, and %made-up parameter keys. Defined in Section 4.2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human judges were used to validate the quality of the synthetic test dataset (Section 4.1), but no human evaluation of the system's generated DSL outputs was performed. All evaluation of system outputs is automated via compiler checking and similarity metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes a separate test set of 1000 samples distinct from the 67k training samples. The test set was generated independently and verified by human judges."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported only as overall averages across the 1000-sample test set. No breakdown by API category, query complexity, or DSL length is provided, despite the paper mentioning roughly 700 APIs with varying frequencies (head vs. tail)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses cases where SFD increases hallucination rates ('adding too many API descriptions can confuse rather than help the LLM'). Section 6 discusses where the fine-tuned model struggles with syntax vs. where RAG struggles with hallucination. Appendix A.1 shows an example with a hallucinated API call."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows that SFD increases hallucination rates (e.g., +6.76 pts for made-up parameters). Section 5.3 discusses how 'the approach to simply add semantically similar API metadata for a query is not useful for DSL generation.' The paper reports that RAG still lags behind fine-tuning on hallucination metrics."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims RAG achieved parity on similarity metric (supported by Table 4 showing 0 delta), RAG was 2 pts better on compilation (supported by Table 4 showing -5.3 delta in unparsed flows), and hallucination lagged by 1 pt for API names and 2 pts for parameters (supported by Table 4 showing +1.7 and +1.11). All claims match the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study design uses controlled single-variable manipulation (varying one factor at a time: number of few-shots, retrieval model, grounding type) which is adequate for the causal claims made about which components contribute to performance (e.g., 'few-shot examples have been successfully teaching the correct syntax')."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title says 'DSL Code Generation' broadly but results are limited to one specific automation DSL with ~700 APIs at Microsoft. Claims about RAG vs fine-tuning are specific to this DSL but the paper draws conclusions about 'NL2DSL generation' generally without adequately bounding them to this specific domain."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, it does not consider whether the fine-tuned model's advantage in hallucination could be due to the specific data distribution rather than the approach itself, or whether GPT-4 vs Codex differences confound the RAG vs fine-tuning comparison."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4 (with 16k token limit)' and 'Codex base model' without specifying exact model versions, snapshot dates, or API versions. No version string like 'gpt-4-0613' is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes 'metaprompt instructions' and 'metaprompt tuning' but does not provide the actual prompt text used. Only a high-level system architecture diagram (Figure 1) shows that prompts include grounding info, few-shot examples, and API definitions, without the actual text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters are reported for GPT-4. For the Codex fine-tuning, LoRA-based approach is mentioned but no learning rate, epochs, LoRA rank, or other hyperparameters are specified. For BERT fine-tuning, only the loss function is described without training hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The system is a single-pass RAG pipeline (retrieve, augment prompt, generate) without agentic scaffolding such as retry logic, multi-step planning, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes sampling 700 publicly available APIs and synthetically generating NL prompts using GPT-4 (Section 4.1), but important preprocessing details are missing: how workflows were sampled from users, filtering criteria, how the 67k training samples were generated vs the 1000 test samples, and what 'manual approval' entailed for test set validation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper jumps from Results (Section 5) to Conclusion and Future Work (Section 6) without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. The paper does not address potential issues like synthetic test set quality, model version effects, or whether results would hold for different DSLs."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'we constrained the length of sequence to 5 APIs' (Section 1) but does not explicitly state what the results do NOT show or what settings are excluded. There is no discussion of what claims the authors are NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the raw workflow data, synthetic NL-DSL pairs, nor the test set are publicly available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we had many samples of workflow automations created by users' (Section 4.1) and sampled 700 publicly available APIs, but does not describe the specific collection procedure, time period, selection criteria for user workflows, or how 'publicly available' was determined."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for a study. Human judges validated test set quality but this is dataset construction, not a human subjects study. The data comes from a standard product usage database."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from user workflows to synthetic NL-DSL pairs to final test/training sets is described at a high level but key steps are missing: how many workflows were initially available, how many were filtered and why, how the split between training (67k) and test (1000) was determined, and what percentage of test samples were validated by human judges."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. The authors are from Microsoft Corporation but no explicit funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed as 'Microsoft Corporation, Redmond, USA' on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The authors are Microsoft employees evaluating approaches using Microsoft's Azure AML infrastructure and a DSL for Microsoft's automation platform (likely Power Automate). Microsoft has a financial interest in the outcome of research that validates their workflow automation technology. No independence from funder is established."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 and Codex models without stating their training data cutoff dates. Since the test set contains synthetic NL-DSL pairs, it is unclear whether GPT-4 could have been exposed to similar API documentation or workflow patterns during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of potential overlap between GPT-4's training data and the test set is discussed. The DSL is custom but the APIs are publicly documented, creating potential contamination risk."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not address whether GPT-4 may have seen documentation for the 700 publicly available APIs used in the benchmark. Since these are public APIs, their documentation was likely in GPT-4's training data, which could give GPT-4 an advantage in understanding function names and parameters."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in a study. Human judges validated test set quality but were not study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted. Human judges validated data quality, which does not constitute a human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time per example is reported, despite the system making multiple GPT-4 calls with varying numbers of few-shot examples (5 vs 20) and API function definitions."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours for fine-tuning, total API spend, or hardware specifications are reported. The paper mentions using Azure AML pipelines but provides no cost or compute details."
       }
     }
   },
   "claims": [
     {
       "claim": "Fine-tuned Codex model scores best on code similarity metric for DSL generation.",
       "evidence": "Table 4 uses fine-tuned model as baseline; RAG variants show 0 or -0.01 delta in average similarity, confirming parity but not superiority for RAG.",
       "supported": "moderate"
     },
     {
       "claim": "Optimized RAG achieves parity with fine-tuned model on similarity metric.",
       "evidence": "Table 4 shows TST + FD achieves 0 delta on average similarity compared to fine-tuned baseline.",
       "supported": "moderate"
     },
     {
       "claim": "RAG-based method has 2 pts better compilation rate than fine-tuned model.",
       "evidence": "Table 4 shows TST + FD has -5.3 delta on %Unparsed flows compared to fine-tuned baseline (abstract says 2 pts, table shows 5.3 pts improvement, slight discrepancy).",
       "supported": "weak"
     },
     {
       "claim": "Hallucination rate for RAG model lags behind fine-tuned model by 1 pt for API names and 2 pts for parameter keys.",
       "evidence": "Table 4 shows TST + FD has +1.7 for made-up API names and +1.11 for made-up parameters vs fine-tuned baseline. Abstract says 1 pt and 2 pts respectively, which is approximately consistent but not exact.",
       "supported": "moderate"
     },
     {
       "claim": "Adding more few-shot samples (20 vs 5) improves performance across all metrics.",
       "evidence": "Table 1 shows improvements in all four metrics when increasing from 5 to 20 few-shots for both Pre-Trained and TST models.",
       "supported": "moderate"
     },
     {
       "claim": "Adding semantically similar API metadata (SFD) can confuse rather than help the LLM.",
       "evidence": "Table 4 shows TST + SFD increases hallucination of parameter keys by +6.76 pts compared to fine-tuned baseline, worse than TST + FD (+1.11 pts).",
       "supported": "moderate"
     },
     {
       "claim": "RAG hallucination rate for API names dropped by 6.29 pts with optimizations.",
       "evidence": "Section 6 cites Table 2, which shows TST + FD achieves -6.29 delta in made-up API names compared to Pre-Trained baseline without FD.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The paper compares fine-tuning (Codex with LoRA) and optimized RAG (GPT-4 with TST-based few-shot retrieval and API function definitions) for DSL code generation across ~700 APIs. Optimized RAG achieves parity with fine-tuning on code similarity but has slightly higher hallucination rates for API names and parameters. RAG with API function definitions significantly reduces parsing errors compared to fine-tuning. Adding too many semantically similar API definitions (SFD) can increase hallucination rather than reduce it.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are reported as single point estimates with no confidence intervals, error bars, standard deviations, or significance tests. It is impossible to assess whether the reported differences (often 1-3 percentage points) are meaningful or within noise."
     },
     {
       "flag": "Confounded comparison: GPT-4 vs Codex",
       "detail": "The RAG approach uses GPT-4 while the fine-tuned model uses Codex. These are fundamentally different models with different capabilities, making it impossible to attribute performance differences to RAG vs fine-tuning rather than to the underlying model quality."
     },
     {
       "flag": "Abstract-results discrepancy",
       "detail": "The abstract states RAG is '2 pts better' on compilation rate, but Table 4 shows a 5.3 pt improvement. The abstract says hallucination lags 'by 2 pts for API parameter keys' but Table 4 shows +1.11 pts. These inconsistencies undermine confidence in the reporting."
     },
     {
       "flag": "No reproducibility artifacts",
       "detail": "No code, data, prompts, or hyperparameters are released. The DSL, API definitions, training data, and test set are all proprietary to Microsoft, making independent verification impossible."
     },
     {
       "flag": "Company evaluating own product",
       "detail": "Microsoft employees evaluate approaches for Microsoft's automation platform using Microsoft's Azure infrastructure. No conflict of interest disclosure is provided."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations, threats to validity, or scope boundaries. No consideration of alternative explanations for the observed results."
     },
     {
       "flag": "Results reported only as deltas",
       "detail": "All tables report delta improvements from baselines rather than absolute values. The actual baseline performance numbers are never stated, making it impossible to assess the absolute quality of any approach."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational work on code generation evaluation (Codex/HumanEval) - central to assessing LLM code generation capabilities."
     },
     {
       "title": "Synchromesh: Reliable code generation from pre-trained language models",
       "authors": ["Gabriel Poesia", "Oleksandr Polozov"],
       "year": 2022,
       "arxiv_id": "2201.11227",
       "relevance": "Introduces TST-based few-shot retrieval for improving code generation reliability, directly used as a technique in this paper."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code generation model relevant to understanding the landscape of code LLMs and their evaluation."
     },
     {
       "title": "Gorilla: Large Language Model Connected with Massive APIs",
       "authors": ["Shishir G. Patil", "Tianjun Zhang"],
       "year": 2023,
       "arxiv_id": "2305.15334",
       "relevance": "Addresses LLM integration with large API sets, directly relevant to the challenge of API function name hallucination in code generation."
     },
     {
       "title": "PAL: Program-aided Language Models",
       "authors": ["Luyu Gao", "Aman Madaan"],
       "year": 2023,
       "arxiv_id": "2211.10435",
       "relevance": "Explores using code generation for reasoning tasks, connecting program synthesis with tool use - relevant to LLM programming capabilities."
     },
     {
       "title": "Improving ChatGPT Prompt for Code Generation",
       "authors": ["Chao Liu", "Xuanlin Bao"],
       "year": 2023,
       "arxiv_id": "2305.08360",
       "relevance": "Studies prompt engineering for code generation with LLMs, directly relevant to prompt-based approaches for improving code quality."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu"],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Foundational work on LLM tool use and integration, relevant to understanding how models interact with external APIs."
     },
     {
       "title": "HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in Hugging Face",
       "authors": ["Yongliang Shen", "Kaitao Song"],
       "year": 2023,
       "arxiv_id": "2303.17580",
       "relevance": "Task orchestration using LLMs across multiple APIs, related to the multi-API planning problem addressed in this paper."
     },
     {
       "title": "TaskMatrix.AI: Completing Tasks by Connecting Foundation Models with Millions of APIs",
       "authors": ["Yaobo Liang", "Chenfei Wu"],
       "year": 2023,
       "arxiv_id": "2303.16434",
       "relevance": "Addresses scaling LLM tool use to massive API sets, directly relevant to the challenge of DSL generation over hundreds of APIs."
     },
     {
       "title": "An Empirical Evaluation of GitHub Copilot's Code Suggestions",
       "authors": ["Nhan Nguyen", "Sarah Nadi"],
       "year": 2022,
       "doi": "10.1145/3524842.3528470",
       "relevance": "Empirical evaluation of AI code generation quality, relevant to understanding how LLM-generated code is assessed."
     }
   ]
 }