ai-research-survey

calibration.json (15745B)

---

 {
   "paper_slug": "adaplanner-adaptive-planning-2023",
   "total_questions": 50,
   "agreement_count": 49,
   "disagreement_count": 1,
   "agreement_rate": 0.98,
   "disagreements": [
     {
       "category": "statistical_methodology",
       "question": "effect_sizes_reported",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "The paper reports percentage improvements with baseline context: '3.73% improvement' from 88.06% to 91.79% (ALFWorld) and '4.11% improvement' from ~88.76% to 92.87% (MiniWoB++). The schema description explicitly states that 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' Tables 2 and 3 provide the raw baseline scores alongside AdaPlanner scores, giving readers full effect magnitude context.",
       "sonnet_justification": "The paper reports raw performance differences (e.g., 3.73% and 4.11% improvement) but does not report standardized effect sizes such as Cohen's d. The raw absolute differences are mentioned but framed without statistical context.",
       "direction": "opus_generous"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract explicitly provides a GitHub URL: 'The implementation of AdaPlanner is available on https://github.com/haotiansun14/AdaPlanner.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available standard benchmarks: ALFWorld and MiniWoB++. No proprietary data was collected or modified."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed library version listing is provided. The appendix describes experimental setup at a high level but lacks sufficient detail to recreate the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a GitHub link is provided, the paper itself contains no step-by-step reproduction instructions, no README with commands, and no 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2 and 3 report success rates as point estimates only (e.g., '91.79%'). No confidence intervals, error bars, or ± notation appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AdaPlanner 'outperforms state-of-the-art baselines by 3.73% and 4.11%' but provides no statistical significance tests (no p-values, t-tests, bootstrap tests, or equivalent)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: 3.73% improvement from 88.06% to 91.79% in ALFWorld and 4.11% improvement in MiniWoB++. Tables 2 and 3 provide the raw baseline scores alongside AdaPlanner scores. Per the schema example, percentage improvement with baseline context counts as YES."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 134 ALFWorld tasks and 53 MiniWoB++ tasks are described but there is no justification for why these sample sizes are sufficient and no power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2 and 3 are single point estimates. No mention of multiple runs, standard deviation, IQR, or any spread measure. Result stability is impossible to assess."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines: BUTLER, ReAct, Reflexion (ALFWorld) and CC-Net, WGE, WebN-T5-3B, RCI (MiniWoB++), covering both training-based and prompting-based methods."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporaneous work from 2023 (Reflexion, RCI, ReAct) alongside older training-based methods, representing reasonable state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4c ablates the code interface (CI) and Figure 4d ablates skill discovery (SD), each showing the contribution of individual components via controlled removal."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only success rate (%) is used. Section 8.1 states 'we use success rate (%) to evaluate the performance of tested methods' and no additional metrics are reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Evaluation is entirely automated via task success/failure in simulated environments (ALFWorld, MiniWoB++). Human evaluation is irrelevant to these automated benchmark claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on standard benchmark test sets (134 ALFWorld tasks, 53 MiniWoB++ tasks). Few-shot demonstrations used for prompting are separate from evaluated tasks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-task-type results for all six ALFWorld types. Table 3 separates MiniWoB++ into feedback/no-feedback subsets. Appendix Table 5 gives per-task results for all 53 MiniWoB++ tasks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Appendix 8.4 shows trajectory examples where baselines fail due to hallucination, but there is no error analysis or discussion of where AdaPlanner itself breaks down. The 47.06% success on 'Pick two' is not analyzed. No systematic failure case discussion for the proposed method."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies (Figures 4c, 4d) show performance drops when components are removed. Table 2 shows gpt-3.5-turbo underperforms text-davinci-002, which is explicitly discussed in Section 4 as an unexpected finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 3.73% and 4.11% improvement with 2x and 600x fewer samples are supported by Tables 2, 3, and Figure 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims via ablation studies (Figures 4c, 4d) are supported by controlled single-variable manipulation — removing code interface or skill discovery individually and measuring the impact."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on two specific text-based simulation environments but uses broad language like 'sequential decision-making tasks' without bounding claims to these specific environments. The title and framing suggest general applicability."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for results are discussed. For instance, performance differences could stem from prompt engineering quality rather than the closed-loop mechanism, but this is not addressed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'gpt-3.5-turbo', 'text-davinci-002', and 'text-davinci-003' as specific model API identifiers in Tables 2 and 3 and Section 8.2. text-davinci-002 and text-davinci-003 are fixed model versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix 8.3 provides full prompt text for all stages (basic_info, initial_planning, code_check, refinement, start_from) with actual expert trajectory examples for all six ALFWorld task types and MiniWoB++ tasks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other LLM sampling hyperparameters are reported anywhere in the paper or appendix."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: planner/refiner dual roles, in-plan and out-of-plan refinement, ask_LLM() action, assertion-based error detection, refine-then-resume mechanism, and skill discovery memory are all described in Sections 3.1-3.3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.1 describes the environments, task selection (9 MiniWoB++ tasks with feedback, 53 tasks from RCI), and demonstration allocation (Table 4). The skill discovery pipeline is described in Section 3.3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Conclusion and Limitations' includes a dedicated limitations discussion noting that AdaPlanner still requires few-shot expert demonstrations for complex tasks."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions only one limitation (need for few-shot demonstrations). No specific threats to validity are discussed such as benchmark contamination, sensitivity to prompt phrasing, or the limited scope of the two test environments."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No bounding of results to text-based environments or these specific benchmarks. Broader applicability is implied without explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental logs, trajectories, or per-episode results are provided for independent verification. Only aggregate success rates are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.1 describes the environments, task types, and how demonstrations were collected (38 human-written + 21 discovered by skill discovery for MiniWoB++, 6 expert samples for ALFWorld)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The evaluation uses automated environments with no human subjects. Standard benchmark data source makes this NA."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is documented: how tasks are selected, how episodes are run, how success/failure is determined (Section 8.1), and how skill discovery produces additional examples (Section 3.3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors are affiliated with Georgia Institute of Technology, as clearly stated on the first page. No authors appear to be affiliated with OpenAI or other evaluated model providers."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed. The schema indicates NA if unfunded. All authors are university researchers with no disclosed external funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial disclosure statement in the paper. Absence of disclosure is not absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-3 (text-davinci-002), GPT-3.5 (text-davinci-003, gpt-3.5-turbo) but does not state the training data cutoff dates for any of these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether ALFWorld or MiniWoB++ benchmarks or their task descriptions may have appeared in the GPT-3/3.5 training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ALFWorld (2021) and MiniWoB++ (2018) were both published before GPT-3/3.5 training data cutoffs. The paper does not address the possibility that task descriptions or solutions appeared in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or wall-clock time per episode are reported. The paper mentions reducing LLM calls as a benefit but does not quantify actual inference costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, GPU hours, or API spend is reported. The computational resources required to run the experiments are not quantified."
       }
     }
   }
 }