ai-research-survey

scan.json (27147B)

---

 {
   "paper": {
     "title": "BioPlanner: Automatic Evaluation of LLMs on Protocol Planning in Biology",
     "authors": [
       "Odhran O'Donoghue",
       "Aleksandar Shtedritski",
       "John Ginger",
       "Ralph Abboud",
       "Ali Essa Ghareeb",
       "Justin Booth",
       "Samuel G Rodriques"
     ],
     "year": 2023,
     "venue": "EMNLP 2023",
     "arxiv_id": "2310.10632"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link (https://github.com/bioplanner/bioplanner) in the footnote on page 1 and in Appendix E, stating 'The dataset and code for evaluation are available at' the URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The BIOPROT dataset is released alongside the code at the same GitHub repository. Section 3 states 'The dataset can be found in the Supplementary Materials' and Appendix E confirms public availability."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, conda environment, or detailed dependency/version specifications found in the paper. Only the models used (GPT-3.5, GPT-4, Llama-2) and embedding model (text-embedding-ada-002) are named."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While prompts are provided in the appendix and the evaluation framework is described, there are no step-by-step reproduction instructions (e.g., a README with commands to run the experiments). The paper describes the methodology but not how to execute the code."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports standard deviation (mean +/- std over 5 runs) in Tables 4 and 5, but does not report confidence intervals. Standard deviation across runs is a variance measure but not a confidence interval or error bar in the statistical sense. However, the +/- notation in tables effectively serves as error bars. This is borderline; the paper does report uncertainty via std dev across runs, which functions similarly. Setting to false because these are standard deviations, not formal confidence intervals."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-4 'outperforms' GPT-3.5 and that shuffling 'leads to a drop in performance' but provides no statistical significance tests (no p-values, t-tests, or similar). Comparisons are based solely on comparing mean values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context: e.g., GPT-4 achieves 70.6% next step accuracy vs GPT-3.5's 65%, and absolute performance numbers are given for all metrics (Tables 4-6, 9-10). The reader can compute effect sizes from the reported numbers with full context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 100 protocols and experiments use 5 runs, but there is no justification for why 100 protocols or 5 runs were chosen, no power analysis, and no discussion of whether the sample size is adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 4, 5, and 9 explicitly report 'mean and standard deviation over 5 runs' with +/- notation for all main metrics. This provides variance information across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares GPT-4 against GPT-3.5 as a baseline, and additionally evaluates Llama-2-7B (Appendix D, Tables 9-10). It also includes a human benchmark comparison (Appendix F)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-3.5, GPT-4 (2023), and Llama-2-7B (2023) were all contemporary models at the time of publication. These represented the state of the art for the evaluated task."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies conditions: shuffled vs. non-shuffled function ordering and with/without feedback loops (Tables 4, 5), which function as ablation studies showing the contribution of ordering information and feedback mechanisms."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: function accuracy, precision, recall, normalized Levenshtein distance, argument precision/recall, SciBERTScore, and BLEU (Tables 4, 5, 6). At least 5 different metrics per task."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F describes a human benchmarking study where an undergraduate biomedical sciences student completed next step prediction (n=32) and function selection (n=20) tasks. Section 5.5-5.6 also describes real-world validation where a scientist reviewed generated protocols and one was executed in the lab."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 100 protocols appear to be used for evaluation without a clear train/dev/test split. The paper does not describe holding out any subset for tuning or development. Since GPT-4 generates the ground truth pseudocode AND is evaluated as a student model, there is no separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported only as aggregate means across all 100 protocols. There is no breakdown by protocol difficulty, biology subdomain, protocol length, or any other category. Only the qualitative example in Appendix C shows a single protocol."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses poor performance on function retrieval and offers explanations (semantic ambiguity between similar functions like Mix and MixSubstance). Section 3.3 discusses errors in generated pseudocode. Appendix D notes Llama-2 was 'unable to complete' next step prediction."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that feedback loops did not consistently help (Table 5 shows mixed results with feedback), function retrieval performance was 'generally poor' (Section 5.2), GPT-4 as evaluator performed only 'slightly above chance' (Section 5.3), and Llama-2 failed on next step prediction entirely (Appendix D)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are well-supported: the automatic evaluation framework is presented (Sections 3-4), the BIOPROT dataset is introduced (Section 3), GPT-3 and GPT-4 are evaluated (Section 5), external validation via lab execution is shown (Section 5.5-5.6). Claims are appropriately hedged."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are limited and justified. The paper uses controlled single-variable manipulation (shuffled vs. non-shuffled, with/without feedback) across 5 runs. Claims like 'shuffling the input functions consistently leads to a drop in performance' (Section 5.2) are supported by these controlled ablations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states its scope as biology protocols and acknowledges limitations: 'Our work is focused on biology, but could be extended to other fields such as chemistry and materials science' (Section 7). The title and abstract specify 'Biology' explicitly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: for poor function retrieval, they suggest semantic ambiguity between function names (Section 5.2). For GPT-4 evaluator results, they note it's unclear if the model can't distinguish or if generations are actually correct (Section 5.3). For Llama's failure, they discuss training regime differences (Appendix D)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-3.5', 'GPT-4', and 'Llama-2' without specific version identifiers or API snapshot dates. No version strings like 'gpt-4-0613' are provided. Section 5.1 says only 'We explore the performance of GPT-3.5 and GPT-4 from the OpenAI API.'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in the appendix: Figure 6 (pseudofunction/pseudocode generation), Figure 7 (pseudocode prediction), Figure 8 (protocol summarization), Figure 9 (error messages for feedback), Figure 10 (protocol retrieval), Figure 12 (protocol generation from retrieved functions). These appear to be actual prompts, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other API hyperparameters are reported anywhere in the paper. Section 5.1 mentions 'text-embedding-ada-002 embeddings' for nearest neighbor search but no generation hyperparameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The scaffolding is described in detail: Section 3.2 describes the automatic feedback loop for pseudocode generation (detecting undefined functions, Python syntax errors, missing units). Section 5.5 describes the Toolformer-like chain-of-thought agent with protocol search tool, with chain-of-thought output shown in Figure 11."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A documents dataset filtering in detail: automatic filtering criteria (no description, linked files, images, tables, fewer than 3 steps) and manual filtering criteria (not biology-related, poorly written). Section 3.3 documents the manual verification process and edit statistics (Table 3)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is titled 'Limitations' and contains substantive discussion organized into three subsections: 'Use of paid API', 'Additional scientific fields', and 'Misuse'."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section includes specific threats: total API cost of approximately $1000 limits reproducibility for resource-constrained researchers (Section 7), the scope is limited to biology (Section 7), and there are potential misuse concerns with protocol generation. The GPT-4 self-evaluation concern (Section 5.3) is also specific."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope: 'Our work is focused on biology' (Section 7), the dataset is limited to Protocols.io (Section 3.1), and the framework evaluates protocol planning specifically (not general scientific reasoning). The paper does not overclaim beyond these boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The BIOPROT dataset (source protocols, pseudocode, pseudofunctions) is available at the GitHub repository. The original protocols from Protocols.io are publicly accessible. This enables independent verification of the generated data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection: protocols were collected from Protocols.io, a public platform with 9,000+ protocols. Automatic and manual filtering criteria are described in Appendix A. Section 3.2 describes the pseudocode translation process."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "The data source is a standard public platform (Protocols.io), not human participants. The human benchmarking in Appendix F uses one undergraduate student, but this is not a human subjects study. NA because the main data comes from a public benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: Protocols.io collection -> automatic filtering (Appendix A) -> manual filtering -> GPT-4 pseudocode generation (Section 3.2) -> automatic debugging feedback loop -> manual verification by scientist (Section 3.3). Table 3 shows the edit breakdown at the manual verification stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. The authors are affiliated with Align to Innovate, Francis Crick Institute, Future House, and University of Oxford, but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the first page: Align to Innovate, Francis Crick Institute, Future House, and University of Oxford. These are clearly disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. The paper evaluates OpenAI models (GPT-3.5, GPT-4), and authors are from organizations that may have relationships with model providers, but this is not explicitly addressed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper. Some authors are affiliated with 'Align to Innovate' and 'Future House', which may have commercial interests related to the findings, but this is not disclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for GPT-3.5, GPT-4, or Llama-2. This is relevant because the Protocols.io data used in BIOPROT was publicly available before these models were trained, so training data overlap is possible."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Protocols.io protocols (the evaluation data) could have been in the training data of GPT-3.5, GPT-4, or Llama-2. Since Protocols.io is a public platform, this overlap is plausible but unaddressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Protocols.io protocols are publicly available online and were published before the training cutoffs of the evaluated models. The paper does not discuss whether these specific protocols could have been included in training data, despite this being a clear contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper's main evaluation is automated benchmark evaluation. The human benchmarking in Appendix F is a small validation exercise (one student), not a formal human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study is conducted. The human benchmarking involved one undergraduate student as a convenience comparison, not a research study requiring ethics approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study. The single participant is described as 'an undergraduate biomedical sciences student' which is minimal characterization but the study is not a human subjects experiment."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study with participant recruitment. Only one student was used for human benchmarking."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects experiment with conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects experiment requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No formal human subjects study with participants who could drop out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Limitations) states: 'In total, we used approximately $1000 for API calls.' This provides the total cost of the experimental evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The total API spend is stated as approximately $1000 (Section 7). While GPU hours for Llama-2 are not specified, the primary compute cost (API calls) is quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4 outperforms GPT-3.5 on next step prediction, with 70.6% vs 65% accuracy (unshuffled) and 57.0% vs 36.1% accuracy (shuffled).",
       "evidence": "Table 4, Section 5.2: Results reported as mean over 5 runs with standard deviations.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 significantly outperforms GPT-3.5 on protocol generation ordering (normalized Levenshtein distance), while showing similar precision and recall of function calls.",
       "evidence": "Table 5, Section 5.2: GPT-4 achieves Ldn of 0.396 vs GPT-3.5's 0.498 (unshuffled, no feedback). Precision/recall are comparable.",
       "supported": "moderate"
     },
     {
       "claim": "Shuffling input functions consistently leads to a drop in performance across all tasks.",
       "evidence": "Tables 4 and 5: Every shuffled condition shows lower accuracy/higher Ldn than the corresponding unshuffled condition.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4 as an evaluator performs only slightly above chance at distinguishing ground truth from generated protocols.",
       "evidence": "Table 8, Section 5.3: GPT-4 evaluator scores range from 35.6% to 43.9%, near the 50% chance level.",
       "supported": "strong"
     },
     {
       "claim": "An LLM-generated protocol was successfully executed in a real biology laboratory, producing viable E. coli cells.",
       "evidence": "Section 5.5-5.6 and Figure 3: The protocol was reviewed by a scientist and executed, with Figure 3 showing E. coli growth on agar plates vs. control.",
       "supported": "moderate"
     },
     {
       "claim": "59% of GPT-4-generated pseudocode required no manual edits, and the framework enables largely accurate dataset creation.",
       "evidence": "Table 3, Section 3.3: Manual verification showed 59% required no edits, 24% needed 1-3 edits, 17% needed more than 3 edits.",
       "supported": "strong"
     },
     {
       "claim": "Llama-2-7B significantly underperforms GPT-3.5 and GPT-4 in function selection and was unable to complete next step prediction.",
       "evidence": "Appendix D, Tables 9-10: Llama-2-7B shows substantially lower scores across all protocol generation metrics and failed entirely on next step prediction.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "BioPlanner introduces an automatic evaluation framework for LLM protocol planning in biology, converting free-text protocols into pseudocode to enable structured comparison. The BIOPROT dataset of 100 manually-verified biology protocols shows GPT-4 outperforms GPT-3.5 on protocol planning tasks, particularly in ordering steps correctly (measured by Levenshtein distance), while Llama-2-7B substantially underperforms both. Function ordering information (non-shuffled input) significantly aids performance. A generated protocol was successfully executed in a real laboratory, demonstrating practical applicability of the framework.",
   "red_flags": [
     {
       "flag": "Benchmark contamination risk",
       "detail": "Protocols.io data is publicly available online and likely in the training data of GPT-3.5 and GPT-4. The paper does not discuss whether the evaluated models may have seen the evaluation protocols during training, which could inflate performance numbers."
     },
     {
       "flag": "GPT-4 generates its own evaluation data",
       "detail": "GPT-4 is used as the 'teacher' to generate the ground truth pseudocode (Section 3.2), and is then also evaluated as a 'student' on reconstructing pseudocode from its own pseudofunctions. This circular dependency may bias results in favor of GPT-4, as it generated the action space and ground truth in a way that may align with its own capabilities."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims that GPT-4 'outperforms' GPT-3.5 are based on comparing mean values without significance tests. Given the overlapping standard deviations in some metrics (e.g., argument-level metrics in Table 4), some claimed differences may not be statistically significant."
     },
     {
       "flag": "No model versions specified",
       "detail": "GPT-3.5 and GPT-4 are referenced without specific API version identifiers. Since model behavior changes across versions, results may not be reproducible."
     },
     {
       "flag": "Single-sample lab validation",
       "detail": "The real-world lab validation (Section 5.5-5.6) tested only one generated protocol (E. coli culture/cryopreservation), which is a relatively routine procedure. This single demonstration is presented as evidence that 'an LLM generates a protocol that is successfully executed,' but one success does not demonstrate general reliability."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent autonomous scientific research capabilities of large language models",
       "authors": ["Daniil A Boiko", "Robert MacKnight", "Gabe Gomes"],
       "year": 2023,
       "arxiv_id": "2304.05332",
       "relevance": "Demonstrates LLM-driven autonomous scientific experimentation (chemistry), directly relevant to evaluating LLM capabilities in science."
     },
     {
       "title": "ChemCrow: Augmenting large-language models with chemistry tools",
       "authors": ["Andres M Bran", "Sam Cox", "Andrew D White", "Philippe Schwaller"],
       "year": 2023,
       "arxiv_id": "2304.05376",
       "relevance": "LLM agent augmented with chemistry tools for synthesis planning and drug discovery, relevant to agentic AI evaluation."
     },
     {
       "title": "Voyager: An open-ended embodied agent with large language models",
       "authors": ["Guanzhi Wang", "Yuqi Xie", "Yunfan Jiang"],
       "year": 2023,
       "arxiv_id": "2305.16291",
       "relevance": "Open-ended LLM agent with self-improving code generation, relevant to agentic LLM evaluation methodology."
     },
     {
       "title": "Large language models are not fair evaluators",
       "authors": ["Peiyi Wang", "Lei Li", "Liang Chen"],
       "year": 2023,
       "arxiv_id": "2305.17926",
       "relevance": "Demonstrates systematic bias when using LLMs as evaluators, directly relevant to methodology quality of LLM evaluation."
     },
     {
       "title": "Sparks of artificial general intelligence: Early experiments with GPT-4",
       "authors": ["Sebastien Bubeck"],
       "year": 2023,
       "arxiv_id": "2303.12712",
       "relevance": "Major GPT-4 evaluation paper that relies on self-evaluation, relevant to evaluation methodology quality assessment."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessi"],
       "year": 2023,
       "arxiv_id": "2302.04761",
       "relevance": "Foundation for tool-augmented LLMs, directly relevant to agentic AI capabilities and evaluation."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "LLM planning and reasoning methodology, relevant to evaluating multi-step LLM capabilities."
     },
     {
       "title": "Self-Refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "arxiv_id": "2303.17651",
       "relevance": "Self-refinement framework for LLMs, relevant to feedback loop methodology used in this paper and agentic AI evaluation."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "relevance": "Primary technical report for GPT-4, central to understanding the capabilities and limitations of the evaluated model."
     },
     {
       "title": "LLM+P: Empowering large language models with optimal planning proficiency",
       "authors": ["Bo Liu", "Yuqian Jiang", "Xiaohan Zhang"],
       "year": 2023,
       "arxiv_id": "2304.11477",
       "relevance": "Combines LLMs with planning frameworks, relevant to evaluating LLM planning capabilities."
     },
     {
       "title": "Generalized planning in PDDL domains with pretrained large language models",
       "authors": ["Tom Silver", "Soham Dan", "Kavitha Srinivas"],
       "year": 2023,
       "arxiv_id": "2305.11014",
       "relevance": "LLM planning evaluation in formal domains, relevant to methodology for evaluating LLM planning abilities."
     }
   ]
 }