ai-research-survey

scan.json (23943B)

---

 {
   "paper": {
     "title": "Automatic Generation of Benchmarks and Reliable LLM Judgment for Code Tasks",
     "authors": ["Eitan Farchi", "Shmulik Froimovich", "Rami Katan", "Orna Raz"],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2410.21071"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. The benchmark generation engine and LaaJ implementations are described but not released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link or data archive is provided. The generated benchmark data (program descriptions, code, summaries, LaaJ scores) is described but not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification, dependency list, or software versions are mentioned. The paper describes the methodology conceptually but provides no technical setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or runnable scripts are provided. The methodology is described at a high level but the paper does not provide enough specifics to replicate the benchmark generation or LaaJ evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. The paper reports point estimates of accuracy (e.g., 99.54%, 99.85%) with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares two LaaJ models (laaj1 vs laaj2) and claims one is better than the other based on raw accuracy numbers (99.54% vs 99.85%) without any statistical significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper reports raw accuracy percentages and score distributions but no standardized effect sizes or baseline-contextualized improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample sizes (40 program ideas from 4 seeds, 232 valid pairs, 464 total pairs) are not justified. No power analysis or rationale for why these numbers are sufficient is provided. The paper mentions sampling 20-30 points for eyeballing based on CLT (Section 7.2) but this is for manual validation, not for the main experimental results."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across runs. The experiment appears to be a single run with no repetition."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baseline comparisons are included. The paper does not compare its LaaJ approach against existing metrics (BLEU, ROUGE-L, CodeBLEU, CodeBERTScore, ICE-Score) despite mentioning them in Section 1.1. The two LaaJ variants are compared against each other but not against any external baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so contemporaneity cannot be assessed. The paper mentions BLEU, ROUGE-L, CodeBLEU, CodeBERTScore, and ICE-Score in the related work but never compares against them."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The system has multiple components (seed generation, code generation, summary generation, similarity scale, threshold selection) but none are individually ablated to understand their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy (correct identification of true/false pairs) is reported as an evaluation metric. No other metrics such as precision, recall, F1, or agreement measures (e.g., Cohen's kappa) are used."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the LaaJ outputs is conducted. The paper discusses the need for human judgment as the gold standard (Section 7.2, 'direct eyeballing of the data is continually needed') but does not actually perform any human evaluation in the reported experiments. The correctness labels are derived from the self-consistency property (same seed = similar, different seed = dissimilar) rather than human judgment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit separation between development and test data is described. The 464 pairs appear to be used for both developing and evaluating the LaaJ models, with no held-out test set mentioned."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 provide score distributions for the two LaaJ models. Section 6.1 reports separate results for the cluster-based experiments (sorting algorithms and graph traversal) vs. the non-cluster experiments, showing per-category breakdowns of accuracy."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 and Table 2 discuss failure cases where scores were incorrectly labeled ('The scores highlighted in red represent those that were incorrectly labeled as true'). Section 7 ('Pitfalls and how we address them') discusses several failure modes including overfitting, LaaJ being only a proxy, and challenges with realistic programs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that some programs were corrupted due to formatting errors (leaving only 232 valid pairs from the original expected number), and that cluster-based comparisons reduced accuracy (from 100% to 98.44%/99.48%). Section 7 discusses multiple pitfalls and challenges."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are methodological ('we introduce a methodology to generate and evaluate LaaJ implementations, utilizing an automatically generated benchmark') and are supported by the described approach and Section 6 experiments. The abstract does not make unsupported quantitative claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims about its approach enabling 'high quality code task solutions' (abstract) and that the self-consistency methodology creates reliable benchmarks. However, no controlled experiment or rigorous evaluation demonstrates that the approach actually causes better solutions compared to alternatives. The self-consistency check itself is presented as sufficient validation without external verification."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the approach 'is general and will be applicable to any context of LLM tasks in which loops in G can be utilized' (Section 1.2) but experiments only cover code summarization comparisons across three programming languages with small, simple programs. The title and abstract suggest generality ('Code Tasks') while results are limited to summary comparison for simple algorithmic programs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the high accuracy results. The 100% accuracy on non-cluster pairs could be due to the trivial nature of distinguishing summaries of completely different programs. The paper does not consider whether the self-consistency property creates artificially easy evaluation tasks."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "No specific model names or versions are mentioned anywhere in the paper. The paper refers only to 'LLMs', 'LLMt' (tested LLM), and 'LLMs' (strong LLM) without identifying what models were used. Section 6 says 'We tried the experiment with different LLMs' without naming any."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "While the paper provides example seed concepts and program idea expansions (Section 3.1-3.2), the actual prompts used for code generation, summary generation, and LaaJ evaluation are not provided. The similarity scale descriptions (Sections 2 and 4) are provided but the full prompt text sent to the LLM judges is not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, etc.) are reported for any of the LLM calls. The threshold of 4 on the 1-7 similarity scale is mentioned but no LLM inference parameters are provided."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the LaaJ architecture with three layers: Prompt Optimization (dynamic few-shot learning, automated prompt extraction), Optimized Inferencing (batch processing, token allocation, error recovery), and Postprocessing (formatting, analytical postprocessing, error handling). The multi-agent pipeline is described conceptually with the graph G framework."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that 'due to formatting errors, some programs were corrupted, leaving us with 232 valid pairs' (Section 6) but does not describe the filtering criteria used to identify and remove corrupted programs, or how many were removed from the original count."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Pitfalls and how we address them') serves as a limitations section, discussing overfitting (7.1), LaaJ being only a proxy for human judgment (7.2), challenges with realistic programs (7.3), and coverage (7.4)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: overfitting to examples shared with the development team (7.1), the gap between LaaJ proxy and human judgment (7.2), simple programs not representing real-world complexity (7.3), and representativeness of generated data (7.4). These are specific to this study rather than generic boilerplate."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do not show. While Section 7.3 acknowledges that simple programs may not represent realistic programs, the paper does not explicitly bound its claims to the tested languages, program sizes, or task types. The generalization claim in Section 1.2 ('the approach is general') is unbounded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (generated programs, summaries, LaaJ scores, pair labels) is not available. Only aggregate results (accuracy percentages and score distributions in Tables 1 and 2) are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data generation procedure is described in detail in Sections 3.1-3.4: starting from seed concepts, expanding to program ideas, generating code in multiple languages, creating summaries, and formulating comparison pairs. Section 6 describes the specific experiment with 4 seeds, 40 program ideas, and the resulting pairs."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data is entirely LLM-generated from seed concepts."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The conceptual pipeline is described (seeds -> ideas -> code -> summaries -> pairs) but critical details are missing: the original number of programs before corruption filtering, the criteria for identifying corrupted programs, and how the 232 valid pairs were selected from the original generation. The jump from 40 program ideas to 232 valid pairs (expected would be more) is not fully explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. All four authors have IBM email addresses, suggesting this is IBM-funded work, but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are disclosed via IBM email addresses (farchi@il.ibm.com, shmulik.froimovich@ibm.com, rami.katan@il.ibm.com, ornar@il.ibm.com). All four authors are at IBM."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are IBM employees. IBM has a commercial interest in LLM code generation solutions. The paper describes a testing methodology for IBM's code generation tools, making IBM a non-independent funder with a stake in demonstrating the approach works. This conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosures are provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No model training cutoff dates are stated. The paper does not even name the models used, let alone their training data cutoffs. Since the LaaJ models evaluate summaries that could overlap with training data, this is relevant."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. The generated benchmark consists of common algorithmic problems (sorting, graph traversal, matrix multiplication) that are abundantly present in LLM training data, which could inflate the quality of generated code and summaries. This is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the generated benchmark programs (common CS algorithms) may have been seen during model training. While the paper argues fresh benchmarks can be generated to avoid overfitting (Section 7.1), this is about overfitting to specific examples, not about pre-training data contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, token counts, or wall-clock times are reported despite the approach involving multiple LLM calls per benchmark instance (code generation, summary generation, LaaJ evaluation). Section 7.1 mentions 'the cost, mainly of LLM inference time, of generating a fresh sample' but does not quantify it."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, GPU hours, or API costs are reported for any of the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "The LaaJ approach achieves 100% accuracy in identifying true and false summary pairs for non-cluster program ideas.",
       "evidence": "Section 6 reports that both LaaJ models achieved 100% accuracy on 464 pairs (232 true, 232 false), and 100% accuracy on symmetry tests (LaaJ(a,b) = LaaJ(b,a)).",
       "supported": "weak"
     },
     {
       "claim": "For cluster-based program ideas (related but not identical programs), the combined benchmark achieves 99.54% accuracy for LaaJ1 and 99.85% for LaaJ2.",
       "evidence": "Section 6.1 reports these accuracy numbers when combining cluster and non-cluster results across 96 pairwise comparisons per cluster for sorting algorithms and graph traversal algorithms.",
       "supported": "moderate"
     },
     {
       "claim": "The self-consistency approach enables generating reliable labeled data at scale for LaaJ validation.",
       "evidence": "The paper describes the graph-based methodology (Section 3.3) and demonstrates it with one experiment (Section 6). However, the scale is modest (40 program ideas, 464 pairs) and no comparison against human-labeled data is provided.",
       "supported": "weak"
     },
     {
       "claim": "The approach is general and applicable to any context of LLM tasks in which loops in G can be utilized to obtain correctness claims.",
       "evidence": "Section 1.2 states this claim. Only code summarization across languages is demonstrated; no other LLM task domains are tested.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes a graph-based methodology for automatically generating benchmarks to evaluate LLM code-related tasks, using self-consistency properties (cycles in the generation graph) to create labeled data without human annotation. In experiments with 40 program ideas across COBOL, PL/1, Java, C++, and Python, two LaaJ models achieved 99.54% and 99.85% accuracy in distinguishing same-seed vs. different-seed code summaries. The approach is interesting conceptually but the evaluation is limited to one task (summary comparison) with simple algorithmic programs, no human validation ground truth, and no comparison against existing metrics.",
   "red_flags": [
     {
       "flag": "No model identification",
       "detail": "The paper never names any LLM used in experiments. Neither the 'strong' LLM (LLMs) nor the 'tested' LLM (LLMt) is identified. This makes the results impossible to reproduce or contextualize."
     },
     {
       "flag": "No external baselines",
       "detail": "Despite discussing BLEU, ROUGE-L, CodeBLEU, CodeBERTScore, and ICE-Score in the introduction, none of these are compared against. The two LaaJ variants are only compared against each other."
     },
     {
       "flag": "Self-referential evaluation",
       "detail": "The benchmark labels are derived from the self-consistency property (same seed = similar summaries) rather than from human judgment. The paper acknowledges human judgment is needed (Section 7.2) but does not perform any. This means the evaluation measures consistency with a structural assumption, not actual quality."
     },
     {
       "flag": "Trivially easy evaluation task",
       "detail": "Distinguishing summaries of completely different programs (e.g., sorting vs. financial applications) is likely trivial for any LLM judge. The 100% accuracy on non-cluster pairs may not demonstrate meaningful capability. The more interesting cluster results (98-99%) receive less attention."
     },
     {
       "flag": "IBM conflict of interest not acknowledged",
       "detail": "All four authors are IBM employees working on LLM code generation solutions. The paper describes a testing methodology for such solutions without acknowledging the potential conflict of interest."
     },
     {
       "flag": "Extremely small experiment scale",
       "detail": "Only 40 program ideas from 4 seed concepts are used. With formatting errors reducing the valid pairs to 232, the evaluation is very small for the generality of the claims made."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation evaluation, a foundational benchmark in LLM code assessment."
     },
     {
       "title": "CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion",
       "authors": ["Yangruibo Ding", "Zijian Wang", "Wasi Uddin Ahmad"],
       "year": 2023,
       "relevance": "Multilingual code completion benchmark relevant to evaluating LLM code generation across programming languages."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLM capability on real-world software engineering tasks."
     },
     {
       "title": "CodeBLEU: A Method for Automatic Evaluation of Code Synthesis",
       "authors": ["Shuo Ren", "Daya Guo", "Shuai Lu"],
       "year": 2020,
       "relevance": "Code evaluation metric incorporating code structure, relevant to automated assessment of code generation quality."
     },
     {
       "title": "CodeBERTScore: Evaluating Code Generation with Pretrained Models of Code",
       "authors": ["Shuyan Zhou", "Uri Alon", "Sumit Agarwal", "Graham Neubig"],
       "year": 2023,
       "relevance": "Semantic-embedding-based code evaluation metric relevant to LLM code generation assessment methodology."
     },
     {
       "title": "ICE-Score: Instructing Large Language Models to Evaluate Code",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "relevance": "LLM-as-judge approach for code evaluation, directly comparable to the LaaJ methodology proposed in this paper."
     }
   ]
 }