ai-research-survey

scan.json (25651B)

---

 {
   "paper": {
     "title": "AutoP2C: An LLM-Based Agent Framework for Code Repository Generation from Multimodal Content in Academic Papers",
     "authors": [
       "Zijie Lin",
       "Yiqing Shen",
       "Qilin Cai",
       "He Sun",
       "Jinrui Zhou",
       "Mingjun Xiao"
     ],
     "year": 2025,
     "venue": "Preprint, Under Review",
     "arxiv_id": "2504.20115"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The code is available at https://github.com/shoushouyu/Automated-Paper-to-Code' — a working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The Paper2Repo benchmark of eight papers is described but no dataset download or structured benchmark release is provided. The papers are identified by arXiv IDs, but there is no released benchmark artifact (e.g., a dataset repository or structured test suite). PaperBench Code-Dev is a third-party benchmark."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 mentions 'Intel(R) Xeon(R) Platinum 8352V CPU and an NVIDIA A40 (48GB) GPU' but no requirements.txt, Dockerfile, or dependency specifications are mentioned in the paper. There is no listing of library versions or environment setup instructions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README with commands, scripts, or 'Reproducing Results' section is described in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 (Paper2Repo benchmark) reports only point estimates with no confidence intervals or error bars. Table 2 reports standard error for some baselines (e.g., '49.2 ± 14.0' for AutoP2C on PaperBench) but this is only for one of two benchmarks and the ± 14.0 is extremely large relative to the claimed improvement."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AutoP2C outperforms baselines but no statistical significance tests are used. Comparisons are based solely on point estimates in Tables 1 and 2."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides baseline context for its improvements: e.g., '91.4% accuracy compared to 81.8% for o1 (9.6% improvement) and 75.3% for R1 (16.1% improvement)' in Section 4.4.1, and relative performance ratios (89.8% to 122.0% of original). COMPclass and COMPfunc are reported with multipliers (1.9x, 2.1x, etc.)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The Paper2Repo benchmark contains only 8 papers with no justification for why 8 papers are sufficient. No power analysis or discussion of sample adequacy is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 results are single-run with no variance or standard deviation reported. Table 2 reports standard error for PaperBench (49.2 ± 14.0) but this is only for one benchmark. No mention of multiple runs or seeds for the main Paper2Repo results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against OpenAI o1 and DeepSeek-R1 on Paper2Repo (Table 1), and against BasicAgent, IterativeAgent, and PaperCoder on PaperBench Code-Dev (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "OpenAI o1, DeepSeek-R1, and PaperCoder are all from 2024-2025, which are contemporary and competitive baselines for this task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3 and 4 present ablation studies. Table 3 removes individual components (blueprint extraction, multimodal parsing, hierarchical decomposition, iterative feedback) and measures impact. Table 4 ablates modalities (image, table)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses four metrics: absolute performance (accuracy), relative performance (ratio to original), COMPclass (class completeness), and COMPfunc (function completeness), as described in Section 4.1 and Table 1."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. COMPclass and COMPfunc use LLM-as-a-Judge scoring (Section 4.1). The system generates code repositories whose quality could benefit from human assessment, but evaluation is entirely automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The Paper2Repo benchmark of 8 papers was constructed by the authors. There is no separation of dev/test — all 8 papers appear to be used for both development and evaluation. The PaperBench Code-Dev is a separate benchmark but the paper does not discuss whether any tuning was done on it."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-paper breakdowns across all 8 papers for all four metrics. The ablation study (Tables 3-4) also breaks down by individual papers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss failure cases or error analysis for AutoP2C. It notes baselines fail (marked with red crosses), but does not analyze where or why AutoP2C's generated code falls short (e.g., the cases where COMPfunc is low like 18.5% or 31.4%)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment shows AutoP2C outperforming baselines. No approaches that were tried and abandoned or configurations that failed are discussed. The ablation results show degradation when components are removed, but no negative results about the full system."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims AutoP2C 'can successfully generate executable code repositories for all eight papers, while OpenAI-o1 or DeepSeek-R1 can only produce runnable code for one paper.' This is supported by Table 1, which shows the baselines fail on 7/8 papers."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Tables 3-4) makes causal claims about component contributions (e.g., 'removal of any component degrades performance'). The controlled single-variable ablation design, removing one component at a time, is adequate for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Code Repository Generation from Multimodal Content in Academic Papers' broadly, but the evaluation is limited to 8 ML papers in specific subdomains (CV, NLP, graph). No discussion bounds the generalization to other fields, programming languages, or paper types. Section 5 briefly mentions extending to other languages as future work but does not bound current claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For example, the baselines (o1 and R1) are single-model systems while AutoP2C is a multi-agent pipeline with multiple LLM calls — the improvement could partly be due to the additional compute rather than the framework design. No such confounds are discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 mentions 'GPT-4o', 'o1-mini', 'o1', and 'o3-mini' without snapshot dates or API versions. These are marketing names that do not uniquely identify a model version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts in natural language (e.g., 'guiding prompt consisting of three key components' in Section 3.4) but does not provide the actual prompt text used in experiments. No appendix with full prompts is included."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states 'we standardize the decoding temperature to 0 for all temperature-configurable LLMs.' The hardware is specified (Intel Xeon Platinum 8352V CPU, NVIDIA A40 48GB GPU). Token counts are reported in Table 1."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The four-stage pipeline is described in detail in Sections 3.2-3.6: repository blueprint extraction, multimodal content parsing, hierarchical task decomposition, and iterative feedback-driven implementation. The agent roles (coding agent, verification agent), workflow, retry/debugging logic, and inter-component dependencies are well-documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 documents the multimodal content parsing pipeline: OCR with MinerU for PDF-to-markdown conversion, VLM for visual elements, LLM for equations and tables, followed by integration and filtering steps. The paper selection criteria for the benchmark are described in Section 4.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated Limitations or Threats to Validity section. The Conclusion (Section 5) contains a single sentence about future work ('explore the extension of AutoP2C to support additional programming languages beyond Python') but no substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. There is no analysis of potential biases in the benchmark selection, limitations of LLM-as-a-Judge evaluation, or risks of the small sample size."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The future work sentence in Section 5 implicitly acknowledges Python-only scope, but no explicit scope boundaries are stated regarding the types of papers, domains, or complexity levels the framework can or cannot handle."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The generated code repositories, LLM judge scores, and raw experimental outputs are not available for independent verification. Only aggregated metrics in Tables 1-4 are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes how the Paper2Repo benchmark was constructed: papers selected from paperswithcode.com with criteria including publication date (after 2024), availability of GitHub repositories, spanning six ML tasks and three modal types."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. The benchmark consists of academic papers and automated evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the system pipeline is documented, the evaluation data pipeline is not fully transparent. It is unclear how COMPclass and COMPfunc scores were computed in practice — which LLM was used as judge, what prompts were given, and whether there was any filtering or aggregation beyond what is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Science and Technology of China and Johns Hopkins University. The evaluated systems (OpenAI o1, DeepSeek-R1) are from different organizations than the authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of funding disclosure is itself a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o, o1-mini, o1, and o3-mini but does not state their training data cutoff dates. The benchmark papers were published in 2024, and it is unclear whether any of these papers or their code repositories appeared in the training data of the models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmark papers or their existing code repositories could have appeared in the training data of the models used. This is a significant concern since the models might have seen the original code repositories."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Paper2Repo benchmark papers have GitHub repositories that were publicly available before the models' likely training cutoffs. The paper does not address whether the models may have already seen these repositories during training, which could inflate structural completeness scores."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports input and output token counts for each paper processed by AutoP2C (e.g., 852K input / 120K output for the first paper). This provides a proxy for API cost, though no dollar amounts are given."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The hardware is specified (Section 4.3: Intel Xeon Platinum 8352V, NVIDIA A40 48GB) and token counts are reported, but no total API spend, wall-clock time per experiment, or total computational budget is stated."
       }
     }
   },
   "claims": [
     {
       "claim": "AutoP2C can successfully generate executable code repositories for all eight papers, while OpenAI-o1 or DeepSeek-R1 can only produce runnable code for one paper.",
       "evidence": "Table 1 shows AutoP2C achieves non-zero performance across all 8 papers while o1 and R1 fail on 7 papers (marked with red crosses). Section 4.4.1.",
       "supported": "strong"
     },
     {
       "claim": "AutoP2C achieves an average relative performance of 99.5% compared to the original implementations across all eight papers.",
       "evidence": "Table 1 shows relative performance ranging from 89.8% to 122.0%. Section 4.4.1 states the average is 99.5%.",
       "supported": "moderate"
     },
     {
       "claim": "AutoP2C achieves 49.2% replication score on PaperBench Code-Dev, surpassing PaperCoder (44.2%).",
       "evidence": "Table 2 reports AutoP2C at 49.2 ± 14.0% vs PaperCoder at 44.2%. Section 4.4.3.",
       "supported": "weak"
     },
     {
       "claim": "Each component of AutoP2C contributes to the overall performance; removing any degrades results.",
       "evidence": "Table 3 shows ablation results across four papers for all four components. Removing blueprint extraction drops average performance from 83.5% to 51.4%. Section 4.5.",
       "supported": "moderate"
     },
     {
       "claim": "AutoP2C achieves 1.9x and 2.1x improvement in class completeness over o1 and R1 respectively.",
       "evidence": "Section 4.4.2 reports average COMPclass of 65.7% for AutoP2C vs 34.9% for o1 and 31.1% for R1, from Table 1.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AutoP2C is a four-stage multi-agent framework for generating executable code repositories from ML research papers, processing multimodal content (text, diagrams, tables). On the authors' Paper2Repo benchmark of 8 papers, AutoP2C generates executable code for all 8 papers while OpenAI o1 and DeepSeek-R1 succeed on only 1, achieving an average 99.5% relative performance to original implementations. On PaperBench Code-Dev, AutoP2C achieves 49.2% replication score, slightly above PaperCoder's 44.2%. Ablation studies confirm each pipeline stage and modality contributes to performance.",
   "red_flags": [
     {
       "flag": "Tiny benchmark size",
       "detail": "The Paper2Repo benchmark contains only 8 papers, which is extremely small for making general claims about repository generation from academic papers. No justification for why 8 papers are sufficient is provided."
     },
     {
       "flag": "No uncertainty quantification on main results",
       "detail": "Table 1 (the main evaluation) reports single-run point estimates with no error bars, confidence intervals, or variance across runs. Temperature is set to 0, but LLM outputs can still vary. The PaperBench result (49.2 ± 14.0) has a standard error so large it overlaps with PaperCoder's 44.2."
     },
     {
       "flag": "Contamination risk",
       "detail": "The 8 benchmark papers all have public GitHub repositories. The LLMs used (GPT-4o, o1, o3-mini) may have seen these repositories during training. The paper does not address this contamination risk, which could inflate both functional and structural completeness scores."
     },
     {
       "flag": "LLM-as-a-Judge without validation",
       "detail": "COMPclass and COMPfunc are LLM-scored metrics (Section 4.1) but no human validation of these scores is provided. The judge LLM, its prompts, and any calibration are not disclosed, making these metrics difficult to reproduce or verify."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "AutoP2C uses multiple LLM calls across four stages (GPT-4o, o1-mini, o1, o3-mini) while baselines are single-model systems. The comparison conflates framework design benefits with additional compute expenditure. Token counts show AutoP2C uses 300K-1.2M input tokens per paper."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity section. This is concerning given the small benchmark, LLM-scored metrics, potential contamination, and narrow domain coverage."
     },
     {
       "flag": "Missing prompt details",
       "detail": "The actual prompts used for each stage are not provided, only natural language descriptions of what they do. This makes the framework impossible to fully reproduce even with the code release."
     }
   ],
   "cited_papers": [
     {
       "title": "MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2024,
       "relevance": "Key multi-agent framework for code generation, directly relevant as a baseline approach for agentic software development."
     },
     {
       "title": "Autogen: Enabling next-gen llm applications via multi-agent conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Foundational multi-agent framework for LLM applications, relevant to the survey's coverage of agentic AI systems."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos Jimenez", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Major agentic coding framework that uses agent-computer interfaces, directly relevant to automated software engineering evaluation."
     },
     {
       "title": "CodeAgent: Enhancing code generation with tool-integrated agent systems for real-world repo-level coding challenges",
       "authors": ["Kechi Zhang", "Jia Li", "Ge Li"],
       "year": 2024,
       "arxiv_id": "2401.07339",
       "relevance": "First framework specifically designed for repository-level code generation using agents with tools."
     },
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Rigorous evaluation methodology for LLM code generation, relevant to benchmarking methodology in the survey."
     },
     {
       "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research",
       "authors": ["Giulio Starace", "Oliver Jaffe", "Dane Sherburn"],
       "year": 2025,
       "arxiv_id": "2504.01848",
       "relevance": "Benchmark for evaluating AI's ability to replicate research, directly relevant to agentic AI capability evaluation."
     },
     {
       "title": "Paper2Code: Automating Code Generation from Scientific Papers in Machine Learning",
       "authors": ["Minju Seo", "Jinheon Baek", "Seongyun Lee", "Sung Ju Hwang"],
       "year": 2025,
       "arxiv_id": "2504.17192",
       "relevance": "Directly competing multi-agent framework for the same paper-to-code task, serving as a baseline comparison."
     },
     {
       "title": "Judging llm-as-a-judge with mt-bench and chatbot arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "Foundational work on LLM-as-a-Judge evaluation paradigm, used as basis for the COMPclass and COMPfunc metrics."
     },
     {
       "title": "Agentcoder: Multi-agent-based code generation with iterative testing and optimisation",
       "authors": ["Dong Huang", "Jie M Zhang", "Michael Luck"],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent code generation with iterative testing, relevant to the survey's coverage of agentic coding approaches."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Major reasoning LLM used as a baseline, relevant to LLM capability evaluation in the survey."
     },
     {
       "title": "CodeRefine: A Pipeline for Enhancing LLM-Generated Code Implementations of Research Papers",
       "authors": ["Ekaterina Trofimova", "Emil Sataev", "Abhijit Singh Jowhari"],
       "year": 2024,
       "arxiv_id": "2408.13366",
       "relevance": "Pipeline for improving LLM code generation from papers, directly related to the paper-to-code task."
     },
     {
       "title": "Magis: Llm-based multi-agent framework for github issue resolution",
       "authors": ["Wei Tao", "Yucheng Zhou", "Yanlin Wang"],
       "year": 2024,
       "relevance": "Multi-agent framework for GitHub issue resolution, relevant to agentic software engineering in the survey."
     }
   ]
 }