ai-research-survey

scan.json (25162B)

---

 {
   "paper": {
     "title": "Bridging LLM-Generated Code and Requirements: Reverse Generation technique and SBC Metric for Developer Insights",
     "authors": ["Ahilan Ayyachamy Nadar Ponnusamy"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.07835"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code and datasets are available on GitHub: GitHub Repository' and Section 3.2 mentions 'A reference implementation was developed in Python and is available in the associated GitHub repository.' A GitHub link is referenced, though the URL itself is not fully visible in the extracted text."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code and datasets are available on GitHub: GitHub Repository.' The 90-requirement dataset is claimed to be available in the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, library versions, or dependency details are provided in the paper. The paper mentions PyTorch cos_sim and Sentence Transformers but does not provide version numbers or a reproducible environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While Section 3.2 describes the high-level steps of the reference implementation, there are no step-by-step reproduction instructions (e.g., commands to run, configuration files to use). The paper relies on the reader to find and figure out the GitHub repository."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for SBC scores or any other results. Results are presented as point estimates in graphs (Figures 2, 4) without uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims models perform 'similarly' and trends are 'closely aligned' (Section 4.2) but provides no statistical tests to support these comparative claims. No p-values, t-tests, or other significance tests are used."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper describes SBC score thresholds qualitatively (e.g., 'above 0.55' for high quality, 'above 0.65' for semantically very close) but provides no systematic effect size analysis or baseline context for interpreting differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset consists of 90 requirements with no justification for why 90 was chosen. No power analysis or discussion of whether 90 requirements is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each LLM was run for 3 iterations (Section 4.1), but no standard deviation, variance, or spread measure is reported across these iterations. Section 4.2 states performance was 'consistent with minimal variance' but provides no quantification."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baseline comparisons are included. The paper does not compare SBC score against existing metrics (BLEU alone, CodeBERTScore, pass@k, etc.) on the same dataset to demonstrate SBC's superiority. It only compares different LLMs using SBC."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so contemporaneity cannot be assessed. The related work discusses G-EVAL, ICE-Score, and metamorphic prompt testing but never compares SBC against them experimentally."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The SBC score has three components (semantic similarity, BLEU, completeness) with fixed weights (0.7, 0.1, 0.2), but no ablation study is conducted to show that these weights are optimal or that each component contributes meaningfully. The weights are stated as design choices with only verbal justification."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports three component metrics (semantic similarity, BLEU, completeness) along with the composite SBC score, though these are sub-components of the proposed metric rather than independent evaluation dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is conducted. The paper explicitly acknowledges this gap in Section 5: 'human feedback remains an essential validation mechanism. Future work should incorporate qualitative assessments from software engineers.' This is a significant omission for a metric claiming to align with developer needs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No held-out test set is used. All 90 requirements are used for evaluation without any train/test split. The SBC weights appear to be set a priori rather than tuned, but there is no validation of this design on held-out data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides per-LLM breakdowns (Section 4.1, Figure 2) and the dataset spans multiple technology categories (UI, Data Layer, Business Logic per Section 3.1). The graphs show per-question performance across LLMs."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 discusses 'missing elements' (omitted requirements) and 'extra elements' (hallucinations) as failure indicators. Figure 3 shows a specific example with missing and extra elements. Section 5 discusses common failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. All models are described as performing similarly and consistently. No failed approaches, sub-optimal configurations, or surprising failures are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the metric 'improves the evaluation of AI-generated code' and offers 'a real-time, interpretable scoring system,' but no comparison against existing metrics is provided to demonstrate improvement. The abstract claims the approach benefits 'developers of all experience levels,' but no user study validates this."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'This integration empowers developers at all levels' (Section 4.3) and 'our approach allows developers to quickly evaluate' (Section 5) without any causal evidence. The claim that SBC 'improves the evaluation of AI-generated code' is causal but unsupported by comparative experiments."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to bridge 'LLM-Generated Code and Requirements' generally, but the study uses only 90 requirements, 4 specific open models, and a limited set of technologies. No bounding of generalization is stated. Section 5 mentions future expansion but does not bound current claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. For example, the observation that all LLMs perform similarly could be due to the simplicity of the requirements, the dominance of the semantic similarity weight (0.7), or the small dataset size. None of these alternatives are considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.3 lists models as 'Codellama 13B', 'Qwen2.5-Coder 14B', 'Deepseek Coder 6.7B', and 'Codestral 22B' with quantization levels (Q4 or Q5) but does not specify exact version identifiers, snapshot dates, or which specific quantization variant was used for each model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.2 describes the process as 'passing the generated code back to the LLM with a detailed prompt to reconstruct the requirement' but the actual prompt text is not provided anywhere in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states 'we conducted all tests with the temperature set to zero.' The SBC weight parameters are also specified (0.7, 0.1, 0.2). However, other LLM parameters (max_tokens, top_p) are not mentioned."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a straightforward pipeline: generate code, reverse-generate requirements, compute SBC score. There is no agent loop, tool use, or autonomous decision-making."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 states '90 requirements were curated' across multiple technologies but does not describe the curation process, selection criteria, or how requirements were formulated. The process for converting SBC output to CSV for visualization is mentioned but not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section 5 ('Conclusion and Next Steps') briefly mentions areas for future exploration but does not systematically discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The conclusion mentions that 'human feedback remains an essential validation mechanism' and suggests expanding the dataset, but these are framed as future work rather than current limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what the results do NOT show, what types of code generation are excluded, or what limitations exist in the current evaluation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states results were 'recorded in JSON format' and the abstract claims 'Our code and datasets are available on GitHub.' The JSON output includes detailed per-question SBC scores and component breakdowns, suggesting raw data availability."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 describes the dataset categories (UI, Data Layer, Business Logic) and states 90 requirements were curated, but does not describe who created the requirements, what criteria were used, or how they were validated as representative of real-world development."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. The dataset consists of curated programming requirements, not human subjects data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the pipeline steps: iterate through requirements, invoke LLM for code generation, perform reverse generation, compare using SBC scoring, store results in JSON. Section 3.6 describes the visualization pipeline from JSON to CSV to pivot tables to graphs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The author lists only a personal email address (ahilanp@gmail.com) with no institutional affiliation disclosed. No university, company, or organizational affiliation is provided."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed. The paper appears to be independent work by a solo author with a personal email, suggesting it may be unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is provided anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses four LLMs (Codellama 13B, Qwen2.5-Coder 14B, Deepseek Coder 6.7B, Codestral 22B) but does not state training data cutoff dates for any of them."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states 'To prevent data contamination, as discussed by [16] and [14], a completely new dataset was created instead of relying on pre-existing benchmark datasets.' This directly addresses train/test overlap by creating novel requirements."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses contamination risk by creating a new dataset rather than using existing benchmarks (Section 3.1). This is a meaningful mitigation strategy, as novel requirements would not appear in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or wall-clock time is reported despite running 4 models x 3 iterations x 90 questions (1,080 inference calls minimum). The paper claims integration into development lifecycle but does not quantify the practical cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget, hardware specifications, or total computation time is stated. The paper uses quantized models locally but does not specify what hardware was used or how long experiments took."
       }
     }
   },
   "claims": [
     {
       "claim": "The SBC score combined with reverse generation provides actionable insights for developers by highlighting missing features and hallucinations in LLM-generated code.",
       "evidence": "Section 4.1 shows examples of missing and extra elements detected (Figure 3). The paper demonstrates that the method identifies gaps but provides no user study or comparison against alternative approaches to validate 'actionable insights.'",
       "supported": "weak"
     },
     {
       "claim": "All four LLMs perform similarly, with performance trends rising and falling together across different questions.",
       "evidence": "Section 4.2 states 'the line graphs for all LLMs are closely aligned' with reference to Figure 4, but no statistical test of similarity is conducted. Only visual inspection of graphs is offered as evidence.",
       "supported": "weak"
     },
     {
       "claim": "Reverse-generated requirements are of high quality and easy to interpret when SBC scores exceed 0.55.",
       "evidence": "Section 4.2 shows sample outputs (Figures 5, 6) for scores above 0.55 and 0.65. This is anecdotal evidence from individual examples, not a systematic validation. No human judges assessed quality or interpretability.",
       "supported": "weak"
     },
     {
       "claim": "The SBC metric improves the evaluation of AI-generated code compared to traditional metrics like BLEU and ROUGE.",
       "evidence": "The abstract and introduction claim improvement over traditional metrics, but no head-to-head comparison is conducted. The paper cites prior work showing BLEU/ROUGE have weak correlations with human judgment but does not demonstrate that SBC has stronger correlation.",
       "supported": "unsupported"
     },
     {
       "claim": "Creating a new dataset prevents data contamination issues present in existing benchmarks.",
       "evidence": "Section 3.1 states the dataset was created 'to prevent data contamination' rather than using pre-existing benchmarks. This is a reasonable design choice but the claim is only partially supported since no contamination analysis is conducted.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes the SBC (Semantic-BLEU-Completeness) score, a hybrid metric that evaluates LLM-generated code by reverse-generating requirements and comparing them to original specifications. Testing on 4 open-source models (Codellama 13B, Qwen2.5-Coder 14B, Deepseek Coder 6.7B, Codestral 22B) across 90 curated requirements, the paper finds that all models perform similarly with closely aligned trends. The reverse generation approach identifies missing functionality and hallucinations, though no human evaluation or comparison against existing metrics validates SBC's effectiveness.",
   "red_flags": [
     {
       "flag": "No baseline comparison for the proposed metric",
       "detail": "The paper proposes SBC as an improvement over BLEU, ROUGE, and CodeBERTScore but never compares SBC against any of these metrics on the same dataset. Without head-to-head comparison, the claim of improvement is unsupported."
     },
     {
       "flag": "No human validation of the metric",
       "detail": "The paper claims SBC provides 'actionable insights' and benefits 'developers of all experience levels' but includes no human evaluation whatsoever. The paper acknowledges this gap but proceeds to make strong claims regardless."
     },
     {
       "flag": "Arbitrary fixed weights without justification",
       "detail": "The SBC score uses fixed weights (0.7 semantic, 0.1 BLEU, 0.2 completeness) with only verbal justification ('semantic similarity is the most critical factor'). No ablation study, sensitivity analysis, or empirical optimization validates these weights."
     },
     {
       "flag": "Very small dataset with no statistical rigor",
       "detail": "90 requirements with 3 iterations per model, but no error bars, standard deviations, confidence intervals, or significance tests. Claims about model similarity and performance trends are based solely on visual inspection of graphs."
     },
     {
       "flag": "Claims substantially outrun the evidence",
       "detail": "The paper claims the metric 'improves evaluation of AI-generated code' and provides 'a real-time, interpretable scoring system' for enterprise integration, but the evidence consists only of running the metric on a small custom dataset with no external validation."
     },
     {
       "flag": "Solo author with no institutional affiliation",
       "detail": "The paper lists only a personal email (ahilanp@gmail.com) with no institutional affiliation, no acknowledgments, and no competing interests statement. While this is not inherently problematic, combined with the methodological gaps, it raises questions about peer feedback during development."
     }
   ],
   "cited_papers": [
     {
       "title": "Out of the BLEU: How should we assess quality of the code generation models?",
       "authors": ["M. Evtikhiev", "A. Pankevich", "V. Zakharov", "D. Chernobrov", "D. Ustalov"],
       "year": 2023,
       "relevance": "Directly evaluates quality metrics for code generation models, relevant to LLM code evaluation methodology."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces the Codex model and HumanEval benchmark, foundational work for LLM code generation evaluation."
     },
     {
       "title": "CodeBERTScore: Evaluating Code Generation with Pretrained Models of Code",
       "authors": ["S. Zhou"],
       "year": 2023,
       "arxiv_id": "2302.05527",
       "relevance": "Neural-based evaluation metric for code generation using pretrained models, directly relevant to code quality assessment."
     },
     {
       "title": "Expectation vs. Experience: Evaluating the Usability of Code Generation Tools Powered by Large Language Models",
       "authors": ["P. Vaithilingam", "T. Zhang", "E. L. Glassman"],
       "year": 2022,
       "relevance": "User study evaluating usability of LLM code generation tools, relevant to developer productivity and AI-assisted coding research."
     },
     {
       "title": "GitHub Copilot AI Pair Programmer: Asset or Liability?",
       "authors": ["V. Arghavan", "F. Amin", "Z. Michel"],
       "year": 2023,
       "arxiv_id": "2206.15331",
       "relevance": "Evaluates GitHub Copilot's effectiveness for developers of different experience levels, relevant to AI-assisted coding productivity."
     },
     {
       "title": "CodeJudge: Evaluating Code Generation with Large Language Models",
       "authors": ["W. Tong"],
       "year": 2024,
       "arxiv_id": "2410.02184",
       "relevance": "LLM-based code evaluation framework addressing syntactic variations and alternative solutions in generated code."
     },
     {
       "title": "G-EVAL: NLG Evaluation using GPT-4 with Better Human Alignment",
       "authors": ["X. Yang"],
       "year": 2023,
       "arxiv_id": "2303.16634",
       "relevance": "LLM-driven evaluation framework using chain-of-thought, foundational for understanding LLM-based evaluation metrics."
     },
     {
       "title": "ICE-Score: Instructing Large Language Models to Evaluate Code",
       "authors": ["Y. Terry"],
       "year": 2024,
       "relevance": "LLM-based code evaluation metric covering multiple programming languages, directly comparable to SBC approach."
     },
     {
       "title": "Validating LLM-Generated Programs with Metamorphic Prompt Testing",
       "authors": ["X. Wang", "D. Zhu"],
       "year": 2024,
       "arxiv_id": "2406.06864",
       "relevance": "Metamorphic testing approach for validating LLM-generated code, relevant to code quality assurance methodology."
     },
     {
       "title": "The Model Openness Framework: Promoting Completeness and Openness for Reproducibility, Transparency, and Usability in Artificial Intelligence",
       "authors": ["M. White", "I. Haddad", "C. Osborne", "X.-Y. Liu", "A. Abdelmonsef", "S. Varghese"],
       "year": 2024,
       "arxiv_id": "2403.13784",
       "relevance": "Framework for model openness and reproducibility, relevant to transparency in AI research methodology."
     }
   ]
 }