ai-research-survey

scan.json (23969B)

---

 {
   "paper": {
     "title": "ByteSized32Refactored: Towards an Extensible Interactive Text Games Corpus for LLM World Modeling and Evaluation",
     "authors": [
       "Haonan Wang",
       "Junfeng Sun",
       "Xingdi Yuan",
       "Ruoyao Wang",
       "Ziang Xiao"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.23979"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub link is provided in the paper: https://github.com/isle-dev/BYTESIZED32-Refactored (footnote 1 on p.1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The corpus itself (32 refactored text games) is available via the GitHub repository. The original ByteSized32 is also publicly available. The evaluation uses these public corpora."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided in the paper. The paper mentions using Python and OpenAI's API but does not specify library versions or dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The experimental procedure is described at a high level (one-shot prompting with GPT-4o, reflection steps) but there are no specific commands or scripts to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates (e.g., '56.25%', '81.25%') with no confidence intervals or error bars in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., Refactored vs. Original, GPT-4o vs. GPT-4) based solely on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While the paper reports raw percentage differences (e.g., '+20.7%' for winnability), no formal effect size measures are provided. The delta columns in Tables 5-8 show raw differences but lack baseline context for most metrics and no standardized effect sizes are reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample size of N=96 generated games (32 games x 3 seeds) is not justified. No power analysis or discussion of whether this sample size is sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Results appear to be single-run numbers with no indication of variability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The original ByteSized32 corpus serves as the baseline, and results are compared between the Refactored and Original corpora across all four evaluation dimensions (Tables 3-8)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is the original ByteSized32 (Wang et al., 2023), which is the direct predecessor that this work extends. This is the most appropriate baseline for this type of contribution."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper does not separately evaluate the contribution of GameBasic.py abstraction vs. the code modularity optimizations, making it impossible to attribute improvements to specific components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation dimensions are used: Technical Validity (three sub-metrics), Specification Compliance (three sub-metrics), Physical Reality Alignment, and Winnability."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated (Python interpreter checks, GPT-based alignment judgments, automated specification matching). No human evaluation of generated game quality is performed, though the paper makes subjective quality claims about game generation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 'unseen evaluation' task specifications (Section 4.1) — the model generates games for specifications not used as the one-shot example, maintaining separation between demonstration and test."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by evaluation dimension (validity, compliance, alignment, winnability) and by sub-metrics within each dimension (e.g., game initialization, valid actions, runnable game within Technical Validity)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases, particularly GPT-5's output formatting failures (Appendix C, Figure 6), API timeout issues (Appendix D), and the decline in technical validity on the refactored corpus compared to the original."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper honestly reports that ByteSized32Refactored shows worse Technical Validity than the original (Tables 3-4), and that Physical Reality Alignment initially drops with the refactored code. GPT-5 failures are also reported in the appendix."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'a mix of performance' with improvements on two dimensions and decreases on the other two, which is accurately reflected in the results (compliance/winnability improve; validity/initial alignment decrease)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'the hierarchical structure of the refactored code presents new challenges for LLMs' (Section 5) and 'abstraction increases reasoning demands' (Summary of Results). These causal attributions are not adequately justified — the refactored vs. original comparison confounds multiple factors (code length, structure, abstraction level) without controlled manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims relevance to 'LLM World Modeling and Evaluation' broadly, but experiments use only GPT-4o (main) and GPT-4/GPT-5 (appendix). Claims about 'LLMs' are not bounded to the tested models. The abstract says 'extensible' and 'scalable' without bounding to the specific domain of common-sense text games."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section acknowledges that 'observed improvements stem from structural biases rather than genuine advancements in modeling ability' (Limitation 1) and that evaluation metrics have 'limited construct validity' (Limitation 2). These are specific alternative explanations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-4o', 'GPT-4', and 'GPT-5' without specifying snapshot dates or API versions (e.g., 'gpt-4o-2024-05-13'). Marketing names without version identifiers are not sufficient."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in Appendix F for game generation, specification compliance evaluation, physical reality alignment evaluation, reflection, and winnability evaluation. The actual prompt templates with placeholder variables are shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A reports hyperparameters for both GPT-4o (temperature=0.0, top-p=1, frequency-penalty=0.0, presence-penalty=0.0) and GPT-5 (temperature=1, top-p=1, frequency-penalty=0.0, presence-penalty=0.0)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The reflection/self-correction scaffolding is described: the model generates code, receives Python interpreter error output, and regenerates (up to 3 reflection steps). The prompt for reflection is provided in Appendix F. Figure 1 shows the overall pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents how the refactored corpus was created (Section 3.2), including the abstraction of base classes into GameBasic.py, the code modularity optimizations (action_map, string construction), and the resulting code statistics (Table 1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the Conclusion, with four numbered limitations covering structural biases, metric construct validity, reflection vs. ability, and efficiency of the reflection process."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) 'observed improvements stem from structural biases rather than genuine advancements in modeling ability'; (2) metrics 'have limited construct validity, being susceptible to framework biases and coverage gaps'; (3) 'reflection enhances measurability' but 'does not directly improve ability'."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or which settings/models/domains are excluded from its claims. The limitations discuss weaknesses but do not draw explicit scope boundaries around the findings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The generated games, evaluation outputs, and raw experimental data are not made available. Only aggregated results in tables and figures are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the experiment setup: one-shot prompting with randomly selected examples, using task specifications from an unseen evaluation set, with up to 3 rounds of reflection. The evaluation metrics and their measurement procedures are described in Section 4.2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved. The study evaluates LLM-generated text games using automated metrics."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from game generation through evaluation is documented: Figure 1 shows the overview, Section 4.1 describes the generation process, Section 4.2 describes each evaluation metric's measurement procedure, and the reflection loop is documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Johns Hopkins University, Liaoning Technical University, Microsoft Research Montréal, and Central University of Finance and Economics. One author (Xingdi Yuan) is from Microsoft Research, and the paper uses OpenAI products, but the paper does not evaluate Microsoft products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. One author is from Microsoft Research, which could have an interest in LLM evaluation frameworks, but no funding relationship is stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o and GPT-5 to generate code based on the ByteSized32 corpus but does not state the training data cutoff for any model. The original ByteSized32 corpus (2023) could be in GPT-4o's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the original ByteSized32 code (publicly available on GitHub since 2023) appeared in GPT-4o's training data, despite the model being asked to generate similar code."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The ByteSized32 corpus was published in 2023 and is publicly available on GitHub. GPT-4o could have been trained on this code, which would affect the fairness of evaluating code generation based on one-shot examples from this corpus. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token consumption, or wall-clock time is reported despite extensive use of GPT-4o, GPT-4, and GPT-5 APIs with multiple reflection rounds per generated game."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware requirements are stated despite the experiments involving generation and evaluation of 96+ games across multiple models with multiple reflection steps."
       }
     }
   },
   "claims": [
     {
       "claim": "ByteSized32Refactored reduces the corpus from 20k to 10k total lines of Python code compared to the original ByteSized32.",
       "evidence": "Table 1 shows average lines per game dropping from 618.1 to 303.19 (-50.9%), and tokens per game from 6792 to 2896 (Section 3.2.2).",
       "supported": "strong"
     },
     {
       "claim": "ByteSized32Refactored allows substantially more examples to fit in LLM context windows compared to the original.",
       "evidence": "Table 2 shows GPT-4o can fit 44 refactored examples vs. 18 original examples in its 128k context window; Qwen3-32B can fit 9 vs. 4; Llama3.2 can fit 2 vs. 1.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o generates games with worse Technical Validity on ByteSized32Refactored compared to the original ByteSized32.",
       "evidence": "Tables 3-4: After 3 reflections, Runnable Game is 61.46% on Refactored vs. 82.29% on Original; Game Initialization is 85.42% vs. 95.83%; Valid Actions is 70.83% vs. 90.62%.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o generates games with better Specification Compliance and Winnability on ByteSized32Refactored compared to the original.",
       "evidence": "Tables 5-6: After reflection, Winnability is 54% on Refactored vs. 34% on Original; Task-critical actions 93.75% vs. 90.62%. However, Distractors compliance is lower (28.12% vs. 53.12%).",
       "supported": "moderate"
     },
     {
       "claim": "The hierarchical structure of the refactored code presents new challenges for LLMs in code generation.",
       "evidence": "Section 5 Summary: Technical validity drops are attributed to 'abstraction increases reasoning demands and raises the generation barrier.' This is a causal interpretation without controlled experimentation isolating abstraction from other changes.",
       "supported": "weak"
     },
     {
       "claim": "GPT-4o consistently outperforms GPT-4 on the original ByteSized32 corpus.",
       "evidence": "Appendix E.1: Tables 4 and 7 show GPT-4o achieves 82.29% runnable games after reflection vs. GPT-4's 57.3%, with gaps across all technical validity metrics.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "ByteSized32Refactored reduces the original ByteSized32 text game corpus from 20k to 10k lines of Python by abstracting common logic into a GameBasic.py foundation library with 7 base classes. Evaluation with GPT-4o shows mixed results: the refactored corpus leads to worse technical validity (61.46% vs. 82.29% runnable games) but better specification compliance and winnability (54% vs. 34% winnable games after reflection). The paper also reports GPT-5 difficulties with evaluation formatting and API constraints, and finds GPT-4o outperforms GPT-4 across all metrics on the original corpus.",
   "red_flags": [
     {
       "flag": "No statistical testing on comparative claims",
       "detail": "All comparisons between Refactored vs. Original and GPT-4o vs. GPT-4 are based on raw percentage comparisons with N=96 games and no significance tests. Differences could be due to random variation."
     },
     {
       "flag": "No variance or uncertainty reporting",
       "detail": "Results appear to be single-run numbers with no standard deviations, confidence intervals, or repeated trials reported. The stability of the results is unknown."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "The original ByteSized32 corpus has been publicly available on GitHub since 2023. GPT-4o may have been trained on this code, which could affect the fairness of evaluating one-shot code generation from this corpus. This is not discussed."
     },
     {
       "flag": "Causal attribution without controlled experiments",
       "detail": "The paper attributes performance differences to 'hierarchical structure' and 'abstraction increasing reasoning demands,' but the refactoring changed multiple factors simultaneously (code length, structure, abstraction, string operations) without ablation to isolate contributions."
     },
     {
       "flag": "GPT-based evaluation of physical reality alignment",
       "detail": "Physical Reality Alignment is judged by GPT itself, creating a circular evaluation where the same family of models generates and evaluates the outputs. No human validation of these automated judgments is reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, a foundational code generation benchmark relevant to evaluating LLM coding capabilities."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLMs on realistic software engineering tasks in large codebases."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Addresses benchmark contamination in code evaluation, relevant to methodology quality assessment."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Benchmark evaluating compositional reasoning in code generation across libraries."
     },
     {
       "title": "A survey on code generation with LLM-based agents",
       "authors": ["Yihong Dong", "Xue Jiang"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey of LLM-based agent approaches to code generation, directly relevant to the survey scope."
     },
     {
       "title": "ByteSized32: A corpus and challenge task for generating task-specific world models expressed as text games",
       "authors": ["Ruoyao Wang", "Graham Todd"],
       "year": 2023,
       "arxiv_id": "2305.14879",
       "relevance": "The original corpus this paper extends; foundational work on LLM world modeling via text game generation."
     },
     {
       "title": "MultiPL-E: A scalable and extensible approach to benchmarking neural code generation",
       "authors": ["Federico Cassano"],
       "year": 2022,
       "arxiv_id": "2208.08227",
       "relevance": "Multi-language code generation benchmark relevant to evaluating LLM programming capabilities."
     },
     {
       "title": "Large language models for code generation: A comprehensive survey of challenges, techniques, evaluation, and applications",
       "authors": ["Nam Huynh", "Beiyu Lin"],
       "year": 2025,
       "arxiv_id": "2503.01245",
       "relevance": "Comprehensive survey of LLM code generation covering evaluation methodology."
     },
     {
       "title": "CoCo-Bench: A comprehensive code benchmark for multi-task large language model evaluation",
       "authors": ["Wenjing Yin"],
       "year": 2025,
       "arxiv_id": "2504.20673",
       "relevance": "Multi-task code benchmark including code understanding and review, relevant to LLM evaluation methodology."
     },
     {
       "title": "TextWorld: A learning environment for text-based games",
       "authors": ["Marc-Alexandre Côté"],
       "year": 2018,
       "relevance": "Foundational text-based game framework relevant to LLM evaluation in interactive environments."
     }
   ]
 }