ai-research-survey

scan.json (19078B)

---

 {
   "paper": {
     "title": "CRQBench: A Benchmark of Code Reasoning Questions",
     "authors": ["Elizabeth Dinella", "Satish Chandra", "Petros Maniatis"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2408.08453"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive link is provided in the paper. The paper mentions reproducing corporate results for open source release using GitHub PR comments, but no link to released code is given."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the benchmark of 100 C++ code reasoning questions derived from the CodeReviewer dataset, but no download link or repository for CRQBench itself is provided in the paper."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The curation methodology is described at a high level but not with reproducible commands or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., 65/100, precision/recall values) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares LLM classification vs keyword matching and GPT-4 vs Falcon performance without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw numbers and percentages are reported (e.g., 65%, 1.8x reduction). No effect sizes with baseline context beyond simple percentages."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark size of 100 questions and classifier evaluation on 100 comments are not justified. No power analysis or rationale for these sample sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported. Results appear to be from single runs with no spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares LLM-based classification against keyword matching (Table 2) and evaluates both GPT-4 and Falcon on the benchmark."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4 (2024) is a contemporary model. Falcon-7B is used as a secondary comparison. The keyword matching baseline is appropriate for the classification task."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The curation pipeline has multiple components (classifier, query type classifier, rephraser, validator) but their individual contributions are not isolated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Precision, recall, and F1 score are reported for the classification task (Table 2). Accuracy with breakdown by query type (VALUE vs EQUIV) is reported for GPT-4 evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human evaluation is central: manual labeling validates the classifier (Section 3.1), human inspection validates rephrased questions (Section 3.2), and GPT-4 outputs are manually evaluated for correctness and contextual relevance (Section 4)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark construction paper, not a model training paper. There is no train/test split to evaluate."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4 performance is broken down by query type (VALUE: 61%, EQUIV: 70%) in Table 3. Error analysis categorizes 35 incorrect responses into missing context (25), C++ knowledge gaps (5), and logic errors (5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 provides error analysis with three categories of failures. Appendix Figures 8-10 show specific examples of incorrect GPT-4 responses."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Falcon achieved only ~25% accuracy, that the keyword matching approach had significantly more false positives, and that the LLM classifier had low recall (52% on Google)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims GPT-4 produces correct responses for 65 of 100 questions, which matches Table 3. The claim about reducing manual effort is supported by Section 3.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are modest: the LLM-based approach reduces manual effort (supported by comparing pipeline stages). The error categorization attributes failures to specific causes (missing context, C++ knowledge, logic errors) based on manual inspection, which is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only C++ code from the CodeReviewer dataset and evaluates only GPT-4 and Falcon, but the title and introduction frame this as a general 'code reasoning' benchmark without adequately bounding to these specific settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the results. For instance, GPT-4's 65% accuracy could be influenced by prompt design, context window limitations, or dataset characteristics, but these are not explored."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4' and 'Falcon' without specifying exact model versions, snapshot dates, or API versions. The Google internal LLM used for curation is not named at all."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in the appendix for the Code Reasoning Classifier (Figure 12), Query Type Classifier (Figure 13), Edit Generator (Figure 14), Expression Extractor (Figure 15), Equiv Rewriter (Figure 16), Value Rewriter (Figure 17), Validator (Figure 18), and GPT-4 evaluation prompt (Figure 11)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the LLM calls."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-step curation pipeline is described in detail in Section 3 and Figure 4, including the chain-of-thought reasoning, query type classification, rephrasing, and validation steps."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the filtering pipeline: starting from code review comments, classifying with the Code Reasoning Classifier, rephrasing, validating, and human inspection. Table 1 quantifies comment categories. Section 3.3 provides sample counts at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Limitations' discusses two specific limitations: the manual answer extraction approach (5.1) and the size of the target environment requiring 10x more comments (5.2)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: answers are derived by human best-effort inspection rather than symbolic verification (Section 5.1), and the cooperative approach requires >10x more comments due to false negatives (Section 5.2)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to C++ only, to the specific dataset used, or discuss what types of code reasoning are not covered."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark data (100 questions, answers, code contexts) is not made available for download. The underlying CodeReviewer dataset is public, but the curated CRQBench subset is not released with a link."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data collection procedure in detail: sourcing from CodeReviewer dataset GitHub PR comments, classification, rephrasing, and validation steps."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for a study. The human inspectors appear to be the authors themselves. Data comes from a public dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 illustrates the full pipeline. Section 3.3 provides counts: 285 GitHub comments inspected to produce 160 candidates, yielding 100 CRQs. Each transformation step is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Bryn Mawr College and Google/Google DeepMind."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Two of three authors are affiliated with Google/Google DeepMind. The curation technique uses a 'Google code aware LLM.' No discussion of whether Google has a stake in the benchmark results. Funding source is undisclosed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4 is evaluated on the benchmark but no training cutoff date is stated. The CodeReviewer dataset is from 2022 and likely in GPT-4's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4 may have seen the CodeReviewer dataset or similar GitHub PR comments during training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The CodeReviewer dataset was published in 2022, before GPT-4's training. The benchmark derives from public GitHub PRs that could be in GPT-4's training data. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted. Human inspection was part of the curation methodology, not a study of human behavior."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a study context."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a study context."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported for any of the LLM calls (curation pipeline or GPT-4 evaluation)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget or resource usage is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4 produces correct responses grounded in the given context for 65 of 100 CRQBench questions.",
       "evidence": "Table 3 shows 65% overall accuracy, with VALUE queries at 61% and EQUIV queries at 70% (Section 4).",
       "supported": "strong"
     },
     {
       "claim": "The cooperative LLM + human approach reduces manual inspection by 1.8x on GitHub and 3.3x on Google comments.",
       "evidence": "Section 3.3 compares manual (285 GitHub comments) vs cooperative (160 comments) approaches, shown in Figure 5.",
       "supported": "moderate"
     },
     {
       "claim": "The majority of GPT-4 errors (25/35) are due to the model lacking necessary context.",
       "evidence": "Section 4 error analysis categorizes 35 errors: 25 missing context, 5 C++ knowledge gaps, 5 logic errors.",
       "supported": "moderate"
     },
     {
       "claim": "Most code review comments (65% GitHub, 80% Google) are not related to code reasoning.",
       "evidence": "Table 1 shows CRQ-related comments at 35% (GitHub) and 20% (Google), based on manual analysis.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CRQBench is a benchmark of 100 C++ code reasoning questions derived from real code review comments using an LLM-assisted curation pipeline. GPT-4 achieves 65% accuracy on these questions, with most errors (71%) due to missing context rather than reasoning failures. The LLM-assisted curation approach reduces manual inspection effort by 1.8-3.3x compared to fully manual curation, though it requires 10x more source comments due to classifier false negatives.",
   "red_flags": [
     {
       "flag": "Contamination risk unaddressed",
       "detail": "The benchmark is derived from public GitHub PRs in the CodeReviewer dataset (2022), which is likely in GPT-4's training data. No contamination analysis is performed."
     },
     {
       "flag": "Google affiliation with undisclosed internal tools",
       "detail": "Two authors are from Google/Google DeepMind. The curation pipeline uses an unnamed 'Google code aware LLM' and Google internal code reviews. No conflict of interest statement is provided."
     },
     {
       "flag": "Very small benchmark size",
       "detail": "100 questions is small for a benchmark. No justification for this sample size is provided, and no statistical uncertainty is reported for the 65% accuracy figure."
     },
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results are single-run point estimates with no confidence intervals, error bars, or repeated measurements."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, a foundational benchmark for LLM code generation evaluation."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "A realistic benchmark for evaluating LLM code reasoning through software engineering tasks."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "relevance": "Introduces MBPP benchmark for LLM code generation evaluation."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2024,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, the primary model evaluated in this benchmark study."
     },
     {
       "title": "Automating Code Review Activities by Large-Scale Pre-training",
       "authors": ["Zhiyu Li", "Shuai Lu"],
       "year": 2022,
       "arxiv_id": "2203.09095",
       "relevance": "Source of the CodeReviewer dataset used to derive the CRQBench benchmark."
     },
     {
       "title": "Can ChatGPT replace StackOverflow? A Study on Robustness and Reliability of Large Language Model Code Generation",
       "authors": ["Li Zhong", "Zilong Wang"],
       "year": 2023,
       "arxiv_id": "2308.10335",
       "relevance": "Evaluates LLM code generation reliability, relevant to understanding LLM code reasoning limitations."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang"],
       "year": 2022,
       "relevance": "Chain-of-thought prompting technique used in the CRQBench curation pipeline."
     },
     {
       "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
       "authors": ["Xuezhi Wang", "Jason Wei"],
       "year": 2023,
       "arxiv_id": "2203.11171",
       "relevance": "Self-consistency technique used for validation in the CRQBench curation pipeline."
     }
   ]
 }