ai-research-survey

scan.json (19365B)

---

 {
   "paper": {
     "title": "Copilot Evaluation Harness: Evaluating LLM-Guided Software Programming",
     "authors": ["Anisha Agarwal", "Aaron Chan", "Shubham Chandel", "Jinu Jang", "Shaun Miller", "Roshanak Zilouchian Moghaddam", "Yevhen Mohylevskyy", "Neel Sundaresan", "Michele Tufano"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2402.14261"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 states 'Future work on this project involves... open-sourcing our data and evaluation code.' No code is released; it is promised for the future."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same as above — data is promised for future release but not currently available. No download links or repository URLs provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 mentions Node 18+, Python 3.8+, Java JDK 1.8, .NET 6.0/7.0/8.0, and clang for C++, but these are build requirements for the evaluated repos, not a reproducible environment specification for the harness itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a high level but there are no runnable scripts or README with commands."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates (e.g., '83%', '74%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-4 'generally outperforms' GPT-3.5 and Code Llama based solely on comparing percentage numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw percentages are reported but no effect sizes (Cohen's d, odds ratios, etc.) are provided. The prompt improvement is described as '5% in C++ to 11% in Java' but without baseline context in a formal effect size measure."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'hundreds of test cases' and data from 'hundreds of public GitHub repositories' but never states exact sample sizes per language/scenario or justifies why those sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be single-run evaluations with no indication of variance across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three models (GPT-3.5, GPT-4, CodeLlama) are compared against each other across tasks, serving as mutual baselines. The paper also positions itself against HumanEval and BLEU-based evaluations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-3.5, GPT-4, and Code Llama were all state-of-the-art or near-SOTA models at the time of writing (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is presented. The harness has multiple components (prompt design, context provision, parsing) but no systematic ablation of their individual contributions beyond the single prompt change for doc generation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Each scenario uses multiple metrics: e.g., doc uses syntax correctness and format correctness; fix uses syntax correctness and fix rate; workspace uses MRR and end-to-end keyword detection."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is performed. All evaluation is automated (syntax checking, test execution, static analysis, keyword detection)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No mention of held-out test sets or train/test splits. The evaluation uses all collected test cases without separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per language (Tables 1, 2) and per bug type (Table 3 for Typescript fix errors)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figures 7, 8, 10, 11 show concrete failure examples with analysis. Section 5.2 discusses error categories for doc generation (code logic changes, syntax changes, incomplete docstrings, irrelevant docstrings)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports cases where GPT-4 fails while GPT-3.5 succeeds (Figures 8, 9, 10), and discusses GPT-4's more nuanced but failing approaches. C# bug fixing shows GPT-3.5 outperforming GPT-4."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to introduce an evaluation harness with metrics for five scenarios and to provide learnings from evaluating three LLMs. The paper delivers on this, though only two of five scenarios have detailed results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.2.1 claims that adding a prompt instruction 'resulted in a significant improvement' (5-11%), which is a causal claim. However, there is no controlled experiment — confounds like different test cases or model stochasticity are not addressed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Evaluating LLM-Guided Software Programming' broadly, but results are only presented for 2 of 5 scenarios (doc and fix). The paper does not sufficiently bound its claims to the tested scenarios."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper offers one speculative explanation for GPT vs CodeLlama differences (training corpus size, Section 5.1.1) but does not systematically discuss confounds like prompt sensitivity, temperature settings, or data selection bias."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-3.5', 'GPT-4', and 'Code Llama' without specifying exact model versions, snapshot dates, or API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figures 6, 7, 8, 11 show the actual prompts sent to the models for fix scenarios. Figure 9 and 10 show doc prompts. These are concrete prompt texts, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, max tokens, or other sampling parameters used for the LLM API calls."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates VS Code's chat extension as a black box IDE integration. The authors cannot be expected to describe the internal scaffolding of the third-party tool."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes repository filtering criteria (size 1-100MB, build time <10 min, must contain methods) and per-language build requirements. Section 4.2 describes test case creation criteria for each scenario."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion mentions future work but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It acknowledges only presenting 2 of 5 scenarios but does not discuss the limitations of generalizing from those results."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. Data release is promised as future work."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes data collection: public GitHub repos, filtered by language-specific build requirements, size constraints, and ability to build/test. Section 4.2 describes test case generation per scenario."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main evaluation. The telemetry data from Microsoft developers (Section 5.3) is usage data, not a recruited study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from repository selection to test case creation to evaluation is described across Sections 4.1 and 4.2, with filtering criteria at each stage. However, exact counts of repos/methods at each stage are not provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Microsoft, Redmond, USA."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are Microsoft employees evaluating VS Code's Copilot integration. Microsoft has a direct financial interest in favorable results for its Copilot product."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. The conflict between Microsoft authorship and Copilot evaluation is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the three models evaluated (GPT-3.5, GPT-4, Code Llama)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The test cases come from public GitHub repositories. Given that GPT models are trained on public code, potential train/test overlap is a significant concern that is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses public GitHub code that likely existed before the models' training cutoffs. This contamination risk is never addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported despite evaluating three LLMs across hundreds of test cases."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget, GPU hours, or total API spend is mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4 generally outperforms GPT-3.5 and Code Llama on documentation generation and bug fixing tasks.",
       "evidence": "Tables 1 and 2 show GPT-4 achieving higher syntax and format correctness for doc and higher fix rates across most languages (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "The evaluation harness metrics are more robust and information-dense than previous evaluation systems like HumanEval.",
       "evidence": "Section 1 and 3 argue that HumanEval uses simple algorithmic problems while the harness uses real-world codebases with multiple metrics, but no empirical comparison of metric quality is provided.",
       "supported": "weak"
     },
     {
       "claim": "Adding a prompt instruction not to change focal code resulted in 5-11% improvement in doc generation across languages.",
       "evidence": "Section 5.2.1 states this improvement but provides no table, no per-language breakdown of the improvement, and no statistical testing.",
       "supported": "weak"
     },
     {
       "claim": "The evaluation dataset is representative of real-world usage based on embedding similarity with Microsoft developer telemetry.",
       "evidence": "Section 5.3 uses PCA visualization (Figures 12, 13) showing test cases and real usage overlap in embedding space, but no quantitative similarity metric is computed.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces the Copilot Evaluation Harness with five evaluation scenarios for LLM-guided IDE programming, presenting detailed results for documentation generation and bug fixing across six languages. GPT-4 generally outperforms GPT-3.5 and Code Llama, though GPT-3.5 sometimes passes with simpler solutions where GPT-4 fails with more nuanced but incorrect approaches. The harness revealed actionable integration improvements, such as prompt modifications that improved doc generation by 5-11%.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All nine authors are Microsoft employees evaluating Microsoft's VS Code Copilot integration. This conflict of interest is not disclosed or acknowledged in the paper."
     },
     {
       "flag": "No statistical rigor",
       "detail": "All comparative claims are based on raw percentage comparisons with no confidence intervals, significance tests, variance reporting, or sample sizes. It is impossible to assess whether observed differences are meaningful."
     },
     {
       "flag": "Incomplete results",
       "detail": "The paper defines five evaluation scenarios but only reports results for two (doc and fix). The abstract and title suggest a more comprehensive evaluation than is delivered."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "Test cases come from public GitHub repositories that the evaluated models were likely trained on. This fundamental validity concern is never discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations, threats to validity, or scope boundaries."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval benchmark, the primary baseline this paper positions against for code generation evaluation."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the three LLMs evaluated in the harness; represents open-source code-specific models."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "One of the three LLMs evaluated; the strongest proprietary model tested."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu"],
       "year": 2023,
       "arxiv_id": "2308.10620",
       "relevance": "Comprehensive SLR on LLMs for SE covering evaluation methods, relevant as a survey of the field this paper contributes to."
     },
     {
       "title": "CodeXGLUE: A machine learning benchmark dataset for code understanding and generation",
       "authors": ["Shuai Lu", "Daya Guo", "Shuo Ren"],
       "year": 2021,
       "relevance": "Comprehensive evaluation platform for LLMs in SE tasks, a key related benchmark."
     },
     {
       "title": "A survey on evaluation of large language models",
       "authors": ["Yupeng Chang", "Xu Wang"],
       "year": 2023,
       "relevance": "Surveys LLM evaluation methods broadly, providing context for evaluation methodology choices."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F. Liu", "Kevin Lin", "John Hewitt"],
       "year": 2023,
       "relevance": "Relevant to context ordering in IDE prompts, cited as motivation for prompt design considerations."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Foundational prompting technique cited as relevant to IDE integration prompt engineering."
     }
   ]
 }