ai-research-survey

scan.json (24344B)

---

 {
   "paper": {
     "title": "LLM-ProS: Analyzing Large Language Models' Performance in Competitive Problem Solving",
     "authors": ["Md Sifat Hossain", "Anika Tabassum", "Md. Fahim Arefin", "Tarannum Shaila Zaman"],
     "year": 2025,
     "venue": "arXiv (IEEE preprint)",
     "arxiv_id": "2502.04355"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "On 166 ICPC World Finals problems (2011-2024), o1-mini and o1-preview achieved 9.64% and 9.04% accepted rates respectively, while GPT-4o, Mistral Large, and Llama-3.1-405B achieved 0% accepted solutions. Performance dropped on unseen 2024 problems (o1-mini 15.4% → o1-preview 7.7%), suggesting data contamination inflates earlier-year results. The o1 models' chain-of-thought reasoning was identified as the key differentiator.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided in reference [27]: https://github.com/sifat-hossain-niloy/LLMs-Performance-in-ICPC-Problems."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset of 166 ICPC World Finals problems is scraped from the ICPC official website, which is publicly available. The GitHub repo contains scripts and solutions."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency versions are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'publicly available scripts' and 'reproducible submissions' but provides no step-by-step reproduction instructions in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., '25.0% accuracy', '9.64% AC rate') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims o1 models 'significantly outperform others' but provides no statistical significance tests. Comparisons are based solely on raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw accuracy percentages are reported but no formal effect sizes (Cohen's d, odds ratios, etc.). The accuracy differences are presented without baseline context beyond raw numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "166 ICPC problems are used with no justification for this sample size or discussion of whether it is sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single-run results are reported (pass@1 setting). No variance, standard deviation, or multiple-run results are presented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five models are compared against each other: GPT-4o, Mistral Large, Llama-3.1-405B, o1-mini, and o1-preview. The general-purpose models effectively serve as baselines for the o1 family."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All five models tested (GPT-4o, Mistral Large, Llama-3.1-405B, o1-mini, o1-preview) were contemporary state-of-the-art models at the time of the study."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing models as-is with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: accuracy (pass@1), verdict distribution (AC/WA/CE/RE/TLE/MLE), resource utilization (runtime and memory), and temporal comparison (pre/post-2024)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant — correctness is determined by automated judge on Codeforces Gym."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The 2024 ICPC problems serve as a temporal held-out set, tested within 24 hours of release to ensure no contamination."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 and surrounding text provide per-category breakdown (Implementation, Graph Theory, Math, Geometry, Greedy, etc.)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Verdict distribution analysis (Table II, Figure 4) discusses failure modes: compile errors, wrong answers, TLE, runtime errors across all models."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4o, Mistral Large, and Llama-3.1-405B achieving 0% accuracy across all years is a substantial negative result. Performance drops on 2024 data are also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about o1 models outperforming others, significant differences, and dataset contamination impact are all supported by the results in Sections IV-B."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes o1 models' superiority to 'chain-of-thought reasoning and calibration' and 'specialized fine-tuning' without controlled experiments isolating these factors. These are causal claims from observational comparisons of different model architectures."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Performance in Competitive Problem Solving' broadly but only tests ICPC World Finals problems. The abstract mentions 'optimizing LLMs for algorithmic tasks' and 'novel problems' without bounding to ICPC specifically. Section V-B acknowledges external validity limitations but the title and framing remain broad."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B RQ3 discusses three alternative explanations for performance variability: dataset contamination, training methodologies, and reasoning strategies. The threats to validity section (Section V) also discusses alternative factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Codeforces Gym acceptance as a proxy for 'reasoning, accuracy, and efficiency' and 'problem-solving capabilities' without discussing the gap between automated judge verdicts and these broader constructs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified only by marketing names: 'GPT-4o', 'Mistral Large', 'Llama-3.1-405B', 'o1-mini', 'o1-preview'. No API versions, snapshot dates, or specific model IDs are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section III-B describes prompt formatting in natural language ('Standardized Prompt Formatting', 'Template Customization') but the actual prompt text is not provided in the paper or appendix."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported for any of the API calls."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used — models are prompted directly in a pass@1 setting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B describes five preprocessing steps in detail: extraction of problem components, standardized prompt formatting, text cleaning, template customization per model, and manual validation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V 'Threats to Validity' provides a dedicated, substantive discussion covering four types of threats: internal, external, construct, and conclusion validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: data contamination of ICPC problems in training data (internal), ICPC not representing broader software engineering (external), zero-shot testing not leveraging full model capabilities (construct), and Codeforces Gym platform dependency (conclusion)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B explicitly states 'ICPC problems... represent only a subset of the programming challenges faced in broader software engineering' and V-C notes zero-shot testing 'may not fully leverage the models' capabilities.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository [27] contains the generated solutions, and ICPC problems are publicly available. Codeforces submissions are logged for reproducibility."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes scraping 166 problems from ICPC official website, covering World Finals 2011-2024, with details on problem structure (Table I)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data source is a standard public benchmark (ICPC problems)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Sections III-A through III-D: scraping → preprocessing → model testing → solution generation and submission, with tools specified (Selenium, PyPDF2, APIs)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Dhaka (Department of CSE) and University of Maryland, Baltimore County (Department of Information Systems)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper evaluates OpenAI products (GPT-4o, o1 family) without any disclosure about potential relationships."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses contamination risk (Section IV-B RQ3) but never states the actual training cutoff dates for any of the five models tested."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B RQ3 and Section V-A discuss data contamination as a threat, noting ICPC problems may exist in training data, and the 2024 problems were tested within 24 hours of the contest."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses contamination by comparing pre-2024 (potentially contaminated) vs. 2024 (unseen) results, and notes the performance drop as evidence of contamination effects. Section V-A discusses this mitigation strategy."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or per-problem inference costs are reported despite using five commercial/large model APIs across 166 problems each."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware used is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single pass@1 results only. No multiple seeds or runs reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many runs were performed. The pass@1 setting implies single attempts but this is not explicitly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is mentioned. The paper does not report what temperature or sampling settings were used, let alone any search over them."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Template customization per model is mentioned but no justification for how configurations were selected or validated."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite comparing 5 models across multiple years and categories."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party models, not the authors' own system. There are no baseline re-implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The o1 models likely use significantly more compute (chain-of-thought reasoning) than GPT-4o or Llama-3.1-405B, but compute budgets are not compared or discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper assumes ICPC problems are good LLM evaluators without questioning construct validity. No discussion of whether Codeforces Gym verdicts actually measure the claimed 'reasoning, accuracy, and efficiency.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used — all models are tested directly via API in pass@1 setting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses temporal leakage by using 2024 ICPC problems tested within 24 hours of release, and comparing against older problems that may be in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether prompts or problem formatting could leak information not available in a real competition setting."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ICPC problems across years share structural similarities or whether training on earlier ICPC-style problems provides unfair advantage on later ones."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is used. The temporal split (2024 problems) is a prevention strategy, but no membership inference, canary strings, or decontamination checks are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "o1-mini and o1-preview significantly outperform GPT-4o, Mistral Large, and Llama-3.1-405B on ICPC problems",
       "evidence": "Table II shows o1-mini achieved 16 AC and o1-preview 15 AC, while GPT-4o, Mistral Large, and Llama-3.1-405B achieved 0 AC across all 166 problems. Figure 4 shows 9.64% and 9.04% AC rates for o1-mini and o1-preview respectively.",
       "supported": "strong"
     },
     {
       "claim": "Data contamination likely contributes to inflated performance metrics on pre-2024 problems",
       "evidence": "Figure 2 shows o1-preview accuracy drops from 25.0% (2017) to 7.7% (unseen 2024), and o1-mini drops from 25.0% (2017) to 15.4% (2024). Section IV-B RQ3 discusses this pattern.",
       "supported": "moderate"
     },
     {
       "claim": "Chain-of-thought reasoning is the key factor enabling o1 models' superior performance",
       "evidence": "Section IV-B RQ3 attributes success to CoT reasoning and specialized fine-tuning, but this is inferred from comparing models with different architectures — no controlled experiment isolates CoT as the causal factor.",
       "supported": "weak"
     },
     {
       "claim": "o1 models demonstrate superior computational efficiency",
       "evidence": "Section IV-C Key Insights claims 'resource efficiency' but no actual resource utilization data (runtime, memory) is presented in the paper despite being listed as an evaluation metric.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Claims outrun evidence",
       "detail": "The paper claims o1 models show 'superior computational efficiency' (Key Insight #4) but presents no resource utilization data despite listing it as an evaluation metric. Resource usage is mentioned as measured by Codeforces but no data is shown."
     },
     {
       "flag": "No statistical rigor",
       "detail": "Claims of 'significant differences' are made without any statistical tests. Five models are compared across 14 years of problems with only raw percentages — no confidence intervals, significance tests, or effect sizes."
     },
     {
       "flag": "Missing hyperparameters and model versions",
       "detail": "Temperature, top-p, and other API parameters are not reported for any model. Models are identified only by marketing names without API versions or snapshot dates, making reproduction impossible."
     },
     {
       "flag": "Uncontrolled compute comparison",
       "detail": "The o1 models use chain-of-thought reasoning that likely consumes significantly more compute per problem than the other models. Comparing pass@1 across models with vastly different inference budgets is misleading without accounting for compute."
     },
     {
       "flag": "Small effective sample for key claims",
       "detail": "The 2024 unseen dataset contains only 13 problems (based on 15.4% = ~2/13 and 7.7% = ~1/13 for o1-mini and o1-preview). Contamination conclusions are drawn from this tiny sample."
     }
   ],
   "cited_papers": [
     {
       "title": "A survey of large language models",
       "authors": ["W. X. Zhao", "K. Zhou", "J. Li"],
       "year": 2023,
       "arxiv_id": "2303.18223",
       "relevance": "Comprehensive survey of LLM capabilities, architectures, and training methodologies."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational work on LLM code generation evaluation."
     },
     {
       "title": "Competition-level problems are effective LLM evaluators",
       "authors": ["Y. Huang", "Z. Lin", "X. Liu"],
       "year": 2023,
       "arxiv_id": "2312.02143",
       "relevance": "Prior work using competitive programming problems as LLM evaluation benchmarks, discusses data contamination."
     },
     {
       "title": "Towards more realistic evaluation of LLM-based code generation: An experimental study and beyond",
       "authors": ["D. Zheng", "Y. Wang", "E. Shi"],
       "year": 2024,
       "arxiv_id": "2406.06918",
       "relevance": "Addresses evaluation methodology issues in LLM code generation benchmarks."
     },
     {
       "title": "LLM-Based test-driven interactive code generation: User study and empirical evaluation",
       "authors": ["S. Fakhoury", "A. Naik", "G. Sakkas"],
       "year": 2024,
       "arxiv_id": "2404.10100",
       "relevance": "Evaluates interactive code generation with test feedback, relevant to understanding LLM coding capabilities."
     },
     {
       "title": "A Performance Study of LLM-Generated Code on Leetcode",
       "authors": ["T. Coignion", "C. Quinton", "R. Rouvoy"],
       "year": 2024,
       "doi": "10.1145/3661167.3661221",
       "relevance": "Benchmarks LLM code generation efficiency on competitive programming platform."
     },
     {
       "title": "Code generation and algorithmic problem solving using Llama 3.1 405B",
       "authors": ["A. Deroy", "S. Maity"],
       "year": 2024,
       "arxiv_id": "2409.19027",
       "relevance": "Evaluates Llama 3.1 405B on algorithmic tasks, directly relevant to benchmark evaluation methodology."
     },
     {
       "title": "Calibration and correctness of language models for code",
       "authors": ["C. Spiess", "D. Gros", "K. S. Pai"],
       "year": 2024,
       "arxiv_id": "2402.02047",
       "relevance": "Studies LLM calibration and confidence in code generation, relevant to evaluation reliability."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin", "A. Odena", "M. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Early work on LLM program synthesis capabilities and evaluation."
     },
     {
       "title": "Quantifying memorization across neural language models",
       "authors": ["N. Carlini", "D. Ippolito", "M. Jagielski"],
       "year": 2022,
       "arxiv_id": "2202.07646",
       "relevance": "Foundational work on measuring memorization in language models, relevant to data contamination concerns."
     }
   ]
 }