ai-research-survey

scan.json (24203B)

---

 {
   "paper": {
     "title": "CodeInsight: A Curated Dataset of Practical Coding Solutions from Stack Overflow",
     "authors": ["Nathanaël Beau", "Benoît Crabbé"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2409.16819"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The benchmark can be accessed at https://github.com/NathanaelBeau/CodeInsight', providing a GitHub URL for the dataset and code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is publicly available via the GitHub repository linked in the abstract (https://github.com/NathanaelBeau/CodeInsight). The 3,409 examples are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions training on 'an a100 GPU with 40GB memory' and 'half-precision computation (FP16)' but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While the dataset is released, there is no README or 'Reproducing Results' section with commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports mean and standard deviation across five seeds for CodeLLaMa fine-tuning experiments (e.g., '48.9 ± 0.6%' for pass@1). Table 1 also reports standard deviations for dataset statistics."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., CodeLLaMa outperforms Mistral, fine-tuning improves over prompting) but provides no statistical significance tests (no p-values, t-tests, or similar)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute performance numbers with baselines for context. For example, Table 2 shows pass@1 improving from 4.7% to 10.1% for Mistral with the second prompt, and Table 3 shows performance across splits with absolute values. This provides enough context to assess magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of seeds (5) used in the fine-tuning experiments, nor for why 3 models were selected. The dataset size of 3,409 examples is described but not justified via power analysis or similar reasoning."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports standard deviations across five seeds for all metrics (e.g., '52.6 ± 0.8%' for pass@1 on the 40-60 split). However, Table 5 (final results) reports only single numbers without variance, which is inconsistent. Giving credit because Table 3 does report multi-run variance for the key fine-tuning analysis."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates three models (Mistral 7B, CodeLLaMa 13B, Starcoder 15B) and compares prompting vs. fine-tuning configurations, providing multiple baselines in Tables 2, 3, and 5."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The models evaluated (Mistral 7B, CodeLLaMa 13B, Starcoder 15B) were contemporary at submission but the paper does not compare against stronger models like GPT-4 on the full benchmark (GPT-4 is only used on a subset of 812 examples for the contamination analysis). No comparison with other leading code models like DeepSeek Coder or WizardCoder."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates prompting strategies (no prompt, first prompt, second prompt in Table 2) and training/test split sizes (20-80, 40-60, 60-40, 80-20 in Table 3), effectively serving as ablation studies showing which configurations matter."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses pass@1, BLEU score, and codeBLEU score (Section 4.1, Table 3), providing three different evaluation metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs is included. All evaluation is automated via unit tests and BLEU/codeBLEU metrics. Given that the paper claims the dataset mirrors real-world coding tasks, human evaluation of output quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes explicit train/test splits (20-80, 40-60, 60-40, 80-20). The final evaluation in Table 5 uses the 40-60 split with a distinct test set of 1,860 examples."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides per-category breakdowns across Labels (MULTILINE, ASSIGN, COMPLEXTASK, etc.) and Packages (Pandas, Numpy, Regex, etc.) for all three models."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides a detailed error analysis of CodeLLaMa outputs with three specific failure examples (Table 12): argument specification error, regex syntax error, and annotation discrepancy."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the first prompt actually decreased CodeLLaMa's performance from 44.7% to 40.3% (Table 2), and that Mistral's non-code-specific pre-training caused it to struggle. The annotation discrepancy in Appendix E is also a candid negative finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a dataset of 3,409 examples with unit tests, support for fine-tuning and evaluation, reduced contamination confirmed by model performance, and public availability. All are supported in the paper: Section 2 describes construction, Section 4 presents evaluations, Section 4.5 addresses contamination, and a GitHub link is provided."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are modest and supported by controlled experiments. The ablation of prompt strategies (Table 2) isolates the effect of prompts, and the fine-tuning experiments (Table 3) use multiple seeds with controlled train/test splits. The contamination analysis in Section 4.5 uses before/after comparisons on the same examples."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title claims 'Practical Coding Solutions' broadly but the dataset is exclusively Python. While the Limitations section mentions the Python restriction, the title and abstract do not bound the scope to Python-only tasks. The claims about 'aiding developers in common tasks' are broader than what Python-only evidence supports."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 discusses whether GPT-4's performance is due to genuine coding ability versus data memorization/contamination, presenting evidence for both interpretations. The error analysis in Appendix E discusses annotation discrepancies as an alternative explanation for model failures."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to models by name and size only: 'Mistral 7B', 'CodeLLAMA 13B', 'Starcoder 15B', and 'GPT-4'. No specific version identifiers, snapshot dates, or API versions are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The two prompts used are provided verbatim in Section 4.2: 'You are a powerful code generation model. Your job is to convert a given natural language prompt into Python function code and return the result.' and 'Return the Result.' appended to the intent."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 reports fine-tuning hyperparameters in detail: LoRA r=16, alpha=16, dropout=0.05, batch size=128, warmup=100 steps, total=400 steps, learning rate=3e-5, AdamW optimizer, FP16 precision. However, no temperature/sampling parameters for the zero-shot/prompting evaluations are reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models are evaluated in a standard prompt-to-completion setting without tool use, feedback loops, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents the full pipeline: starting with 7,300 raw examples, filtering to 2,707, then annotating to 3,409. Filtering criteria are described in Section 2.2 (authenticity, extractability, alignment, executability), and the annotation process is described in Section 2.3 with specific tasks."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the Conclusion, discussing the dataset's specialized nature, potential annotation biases, scope limitations, and Python-only restriction."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section contains only generic disclaimers: 'specialized nature... may not fully represent the broader spectrum', 'could introduce biases', 'current scope may limit its adaptability'. None are specific threats tied to particular findings or design choices in this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section mentions Python restriction but does not explicitly state what the dataset does NOT show or claim. There is no equivalent of 'our results do not generalize to X' or 'we did not test Y'. The boundaries are implied but not explicitly stated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset is released on GitHub (https://github.com/NathanaelBeau/CodeInsight) including all 3,409 examples with intents, code, and unit tests, allowing independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes data sources in detail: CoNaLa dataset (2,379 hand-written + 3,121 top-ranked unrefined examples), plus 600 additional Stack Overflow samples focusing on Pandas, Numpy, and Regex. Selection criteria include community engagement metrics (votes, views) and temporal weighting."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 describes the annotation team: 'A team of five data science professionals, each with a minimum of five years of experience, contributed to the labeling.' While recruitment channels are not described, the annotator qualifications and time constraints (20 minutes per example, ~12 minutes average, ~540 total hours) are documented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented across Sections 2.1-2.3: 7,300 raw examples → filtering (with specific counts per source: 1,993/5,500 from CoNaLa, 294/600 Pandas, 242/600 Numpy, 178/600 Regex) → 2,707 filtered examples → annotation (3 tasks) → 3,409 final examples (from 2,702 distinct problems, some yielding multiple solutions)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Nathanaël Beau is affiliated with both Université de Paris (LLF, CNRS) and onepoint (a company). Benoît Crabbé is affiliated with Université de Paris."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since no funding is disclosed, it is impossible to assess funder independence. The first author has a corporate affiliation (onepoint) which could represent a potential conflict, but no funding relationship is stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper. The first author's dual affiliation with onepoint (a consulting company) is noted but no explicit financial interest declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates pre-trained models (Mistral 7B, CodeLLaMa 13B, Starcoder 15B, GPT-4) on the benchmark but does not state training data cutoff dates for any of them."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 directly addresses data contamination by comparing GPT-4's performance on original CoNaLa examples versus the rewritten CodeInsight versions. The BLEU score drop from 58.8 to 47.6 after annotation is used as evidence of the rewriting's effectiveness against memorization."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 describes deliberate annotation design to reduce contamination: rephrasing intents, normalizing variable names, converting snippets to function format. Section 4.5 empirically tests contamination by comparing GPT-4 on original vs. rewritten examples. Appendix F provides concrete examples of potential memorization patterns."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in an experimental study. The annotators created the dataset but were not subjects of research."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects. Annotators were professional contributors to dataset construction, not study participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects. Annotator qualifications (5+ years data science experience) are described but this is about dataset construction, not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects; not an experimental study involving human randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects; not an experimental study requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as research subjects."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported for any of the model evaluations. The paper does not mention API costs or wall-clock time for generating predictions."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'an a100 GPU with 40GB memory' for fine-tuning but does not state total GPU hours, training time, or overall computational budget. The annotation effort of 540 hours is mentioned but computational cost is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeInsight comprises 3,409 expert-curated Python examples derived from Stack Overflow, with an average of three unit tests per example.",
       "evidence": "Section 2.3 states the annotation yielded 3,409 examples from 2,702 distinct problems. Table 10 confirms average 3.0 ± 0.4 unit tests across the full dataset.",
       "supported": "strong"
     },
     {
       "claim": "The dataset's annotation process (rephrasing intents, normalizing variables, converting to functions) reduces data contamination, confirmed by GPT-4's BLEU score dropping from 58.8 to 47.6 on rewritten examples.",
       "evidence": "Section 4.5 reports GPT-4 achieved BLEU 58.8 on original CoNaLa examples and 47.6 on the rewritten CodeInsight versions of the same 812 examples. GPT-4 still passed 64% of unit tests.",
       "supported": "moderate"
     },
     {
       "claim": "Fine-tuning with as little as 40% of the dataset matches the performance of training on 60-80% for CodeLLaMa.",
       "evidence": "Table 3 shows pass@1 of 52.6% (40-60), 53.4% (60-40), and 53.1% (80-20) across five seeds, with overlapping standard deviations.",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning improves model performance over zero-shot prompting.",
       "evidence": "Table 2 shows CodeLLaMa at 48.1% with best prompt; Table 5 shows 53.1% after fine-tuning on 40-60 split. Mistral improves from 10.1% to 38.4%.",
       "supported": "moderate"
     },
     {
       "claim": "Appending 'Return the Result.' to the intent leads to overall improvement in performance across all models.",
       "evidence": "Table 2 shows improvements for all three models with the second prompt: Mistral 4.7%→10.1%, CodeLLaMa 44.7%→48.1%, Starcoder 45.1%→46.8%. No significance tests accompany these comparisons.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeInsight introduces a curated dataset of 3,409 Python code generation examples from Stack Overflow, each with expert-refined intents and an average of three unit tests. The annotation process (rephrasing intents, normalizing variable names) demonstrably reduces data contamination, as shown by GPT-4's BLEU score dropping from 58.8 to 47.6 on rewritten examples while still passing 64% of unit tests. Fine-tuning CodeLLaMa on just 40% of the dataset achieves comparable performance to using 60-80%, reaching ~53% pass@1. Per-category breakdowns reveal that complex tasks (COMPLEXTASK, Regex) remain challenging for all models.",
   "red_flags": [
     {
       "flag": "Incomplete variance reporting",
       "detail": "Table 3 reports standard deviations across 5 seeds for training split experiments, but Table 5 (the final results table) reports only single-run numbers without any variance measure, making it impossible to assess result stability for the main findings."
     },
     {
       "flag": "No significance tests for comparisons",
       "detail": "The paper compares three models and multiple prompting strategies, claiming some outperform others, but no statistical significance tests are performed. The improvements could be within noise."
     },
     {
       "flag": "GPT-4 only partially evaluated",
       "detail": "GPT-4 is evaluated only on 812 CoNaLa-derived examples for contamination analysis, not on the full benchmark. This selective evaluation leaves the strongest baseline's full performance unknown, potentially making the other models look more competitive."
     },
     {
       "flag": "Corporate affiliation undisclosed as conflict",
       "detail": "First author is affiliated with onepoint (a consulting company) but no funding disclosure or competing interests statement is provided. The relationship between the company and the research is unclear."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and Codex, foundational for code generation evaluation."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark for code generation evaluation."
     },
     {
       "title": "Measuring Coding Challenge Competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart"],
       "year": 2021,
       "relevance": "Introduced APPS dataset for evaluating code generation on competitive programming problems."
     },
     {
       "title": "DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation",
       "authors": ["Yuhang Lai", "Chengxi Li"],
       "year": 2023,
       "relevance": "Data science code generation benchmark from Stack Overflow, directly compared with CodeInsight."
     },
     {
       "title": "Code LLaMA: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the primary baseline models evaluated on CodeInsight."
     },
     {
       "title": "StarCoder: May the Source Be With You!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "One of the primary baseline models evaluated on CodeInsight."
     },
     {
       "title": "Mistral 7B",
       "authors": ["Albert Q. Jiang", "Alexandre Sablayrolles"],
       "year": 2023,
       "arxiv_id": "2310.06825",
       "relevance": "One of the primary baseline models evaluated on CodeInsight."
     },
     {
       "title": "Learning to Mine Aligned Code and Natural Language Pairs from Stack Overflow",
       "authors": ["Pengcheng Yin", "Bowen Deng"],
       "year": 2018,
       "relevance": "Introduced CoNaLa dataset, the primary data source for CodeInsight's examples."
     },
     {
       "title": "Execution-Based Evaluation for Open-Domain Code Generation",
       "authors": ["Zhiruo Wang", "Shuyan Zhou"],
       "year": 2022,
       "arxiv_id": "2212.10481",
       "relevance": "Introduced ODEX benchmark for execution-based code generation evaluation, compared in Table 4."
     },
     {
       "title": "CodeBLEU: A Method for Automatic Evaluation of Code Synthesis",
       "authors": ["Shuo Ren", "Daya Guo"],
       "year": 2020,
       "arxiv_id": "2009.10297",
       "relevance": "Proposed the codeBLEU metric used as one of the evaluation metrics in CodeInsight."
     }
   ]
 }