ai-research-survey

calibration.json (22003B)

---

 {
   "paper_slug": "adapting-knowledge-prompt-2025",
   "total_questions": 50,
   "agreement_count": 46,
   "disagreement_count": 4,
   "agreement_rate": 0.92,
   "disagreements": [
     {
       "category": "evaluation_design",
       "question": "human_evaluation",
       "sonnet_applies": false,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "Human evaluation IS relevant for APR. Exact Match only catches patches identical to a reference fix; there can be multiple valid fixes. Human evaluation of generated patches would strengthen the claims about repair effectiveness. The paper itself acknowledges EM may not capture the full fix ratio (Construct Validity section). The criterion is applicable — the paper could reasonably have included human evaluation of patches — but chose not to. The schema says applies=false only when 'human evaluation is clearly irrelevant to the claims.' For APR, where correctness of patches matters and automated metrics are known to be incomplete, human evaluation is relevant.",
       "sonnet_justification": "Human evaluation is clearly not applicable here. The paper evaluates automated program repair using automated metrics (EM, SC, CodeBLEU), and human evaluation of generated patches is not necessary for the claims being made about relative performance.",
       "direction": "applies_boundary"
     },
     {
       "category": "claims_and_evidence",
       "question": "alternative_explanations_discussed",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "The Threats to Validity section (Section VI.B) discusses generic threats: randomness in training (Internal Validity), EM metric limitations (Construct Validity), and limited generalizability (External Validity). However, these are methodological limitations, not alternative explanations for the observed results. The paper does not discuss specific alternative explanations such as: whether prompt tuning's advantage is simply due to having more trainable parameters per effective sample, whether the improvement is driven by the prompt providing a stronger inductive bias rather than domain knowledge per se, or whether the 87.33% improvement figure is inflated by the choice of data scarcity threshold. The Code Refinement variable anonymization discussion is a dataset-specific observation, not an alternative explanation for the main findings. Per the schema: 'A threats-to-validity section counts only if it discusses specific alternative explanations for the observed results, not just generic methodological limitations.'",
       "sonnet_justification": "The Threats to Validity section (Section VI.B) discusses specific alternative explanations: randomness in training (Internal Validity), that EM may not capture the full fix ratio, and that results may not generalize to other models/datasets. The discussion also considers why Code Refinement shows lower improvement (variable anonymization hypothesis).",
       "direction": "sonnet_generous"
     },
     {
       "category": "setup_transparency",
       "question": "model_versions_specified",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "The paper specifies model names and parameter counts (CodeT5+ 220M, CodeT5+ 770M, GPT-Neo 1.3B) and states they are loaded from 'official versions available on Hugging Face.' However, it does not specify exact Hugging Face model identifiers (e.g., Salesforce/codet5p-220m) or commit hashes. Per the schema: 'Marketing names like Gemini-2.5 or GPT-4o without a snapshot date or API version do NOT count as specified versions.' CodeT5+ 220M is a model family name with size, not a precise version — model weights on Hugging Face can be updated without changing the model name. These are open-weight models with specific sizes, which makes them more identifiable than API models, but the schema explicitly requires version-level specificity.",
       "sonnet_justification": "Model sizes and versions are specified: 'CodeT5+ 220M (220 million parameters) and CodeT5+ 770M (770 million parameters)' and 'GPT-Neo 1.3B (1.3 billion parameters)'. The paper states models are loaded from official Hugging Face versions.",
       "direction": "sonnet_generous"
     },
     {
       "category": "setup_transparency",
       "question": "prompts_provided",
       "sonnet_applies": true,
       "sonnet_answer": true,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "Tables I and III show prompt TEMPLATES with placeholders ([X], [mask], [bugType], [repairAction], [repairPattern], [SOFT], etc.). Per the schema: 'A prompt TEMPLATE with placeholders (e.g., [Task Description]) does NOT count unless the actual fill values are also provided — the reader must be able to reconstruct every prompt sent to the model.' While [X] (buggy code) and [mask] (output slot) are structural model placeholders, the knowledge prompt templates use domain knowledge placeholders like [bugType], [repairAction], [repairPattern], [AST], [errorMessage], [tags] whose actual fill values are dataset-derived and not fully enumerated. The paper shows one example (Fig. 4) but does not provide the actual domain knowledge values used across all experimental instances. A reader cannot reconstruct every prompt sent to the model.",
       "sonnet_justification": "Full prompt templates are provided in Tables I and III, including all seven basic prompt templates (BP1-BP7) in both hard and soft variants, and example knowledge prompt templates for various domain knowledge types. The templates show exact structure with placeholder positions.",
       "direction": "sonnet_generous"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The Conclusion states: 'Our source code and experimental data are publicly available at: https://github.com/Cxm211/k-prompt'. A working GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses six publicly available APR benchmark datasets (BugsInPy, Code Refinement, Defects4J, ManySStuBs4J-SStuBs, TFix, xCodeEval-APR-C). These are standard public benchmarks with citations and the GitHub repo claims to include experimental data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions OpenPrompt and PyTorch frameworks, and hardware (NVIDIA A40 48GB GPU and H100 80GB GPU), but does not provide a requirements.txt, Dockerfile, or specific library version numbers sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides no step-by-step reproduction instructions, no README walkthrough, and no scripts to replicate experiments. Source code is available on GitHub, but the paper text does not include reproduction guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (Tables VI, VII, VIII, IX, X) are reported as single point estimates with no confidence intervals, error bars, or plus-minus notation. Results are averaged over three runs but no uncertainty is reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported anywhere in the paper. Claims of 'outperforms' and 'improvement' are based solely on comparing raw numbers without p-values, t-tests, or any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports relative improvement percentages (e.g., '87.33% on average') but no standardized effect sizes (Cohen's d, etc.). While tables show both baseline and treatment numbers providing some context, the headline claims use relative percentages without standardized measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper selects 1% of large datasets and uses datasets with 260-1082 training instances but provides no justification for why these sample sizes are sufficient. No power analysis is presented."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly states it runs experiments three times with different seeds and averages results, but never reports standard deviations, confidence intervals, or any spread measure. Only mean values appear in all tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Fine-tuning is the primary baseline, and a 'Naive Copy' baseline (copying buggy code as fix) is also included. Results are compared across all six datasets."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is fine-tuning of the same pre-trained models (CodeT5+, GPT-Neo), which is the standard contemporary approach for APR. This is the appropriate and competitive baseline for evaluating prompt tuning."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 compares hard vs. soft prompts and different prompt templates (BP1-BP7). RQ3 ablates individual types of domain knowledge (repair action, repair pattern, bug type, AST, error message, tags) and their combinations. These serve as ablation studies."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: Exact Match (EM), Syntactically Correct Patch (SC), and CodeBLEU. Section IV.D describes all three."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluation IS applicable for APR — the paper itself acknowledges that EM may not capture all valid fixes (Construct Validity). Human evaluation of generated patches would have strengthened claims about repair quality. No human evaluation was conducted; all evaluation is automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly splits data into training, validation, and test sets in an 8:1:1 ratio (Table IV). Results are reported on held-out test sets. RQ4 uses a fixed 500-instance test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across all six individual datasets, four programming languages, and three models. Tables VI-X provide detailed per-dataset, per-model breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases: Code Refinement's variable anonymization limiting performance, xCodeEval where noisy domain knowledge reduces effectiveness, and cases where combining multiple knowledge types hurts performance."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: combining multiple domain knowledge types does not lead to incremental gains (RQ3/Table IX), certain prompt templates consistently underperform for GPT-Neo, Code Refinement shows limited improvement, xCodeEval knowledge prompts sometimes reduce performance below baseline."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '87.33% average improvement over fine-tuning in data scarcity scenarios' which is supported by Table VI results. The claim about comprehensive evaluation across three LLMs, six datasets, and four languages is confirmed by the experimental design."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims (prompt tuning 'improves' performance). The ablation design is adequate — they control for model architecture, datasets, and evaluation protocol, varying only the tuning method in a controlled single-variable manner."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges: 'Our experimental results may be applicable to specific datasets and models only' (Section VI.B). The conclusion states evaluation is limited to specific datasets, models, and prompt templates."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Threats to Validity section discusses generic methodological limitations (randomness, EM limitations, generalizability) but does not discuss specific alternative explanations for why prompt tuning outperforms fine-tuning. Per the schema, a threats-to-validity section that discusses only generic methodological limitations does not count. The paper does not consider whether the improvement is due to additional inductive bias, effective parameter count differences, or other confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'CodeT5+ 220M', 'CodeT5+ 770M', and 'GPT-Neo 1.3B' with parameter counts. It states models are loaded from 'official versions available on Hugging Face' but does not provide exact Hugging Face model identifiers or commit hashes. Per the schema, marketing names without version/snapshot identifiers do not count as fully specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Tables I and III provide prompt TEMPLATES with placeholders ([X], [mask], [bugType], [repairAction], etc.). Per the schema, 'A prompt TEMPLATE with placeholders does NOT count unless the actual fill values are also provided.' The domain knowledge placeholders' actual values are not provided for all experiments. A reader cannot reconstruct every prompt sent to the model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table V provides a complete hyperparameter table: optimizer (AdamW), Adam epsilon (1e-8), learning rate (5e-5), LR scheduler (Linear), training epochs (10), temperature (1.0), top-p (0.9), beam number (5), repetition penalty (1.0)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is standard prompt tuning and fine-tuning of LLMs without any agent framework, tool use, or complex scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.C documents the preprocessing pipeline: single-hunk fix extraction, 8:1:1 train/val/test split, 1% sampling from large datasets using three seeds with averaging. Table IV shows final dataset statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.B 'Threats to Validity' contains three substantive subsections: Construct Validity, Internal Validity, and External Validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: EM may not capture all valid fixes for THIS paper's APR evaluation, randomness in the 1% sampling of THIS paper's datasets, and results may only apply to the specific datasets and models tested."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The conclusion explicitly states scope: evaluation is limited to 'four datasets, three relatively small pre-trained code models, and limited prompt templates.' External Validity also acknowledges results may be dataset/model specific."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark datasets are publicly available (BugsInPy, Code Refinement, Defects4J, ManySStuBs4J, TFix, xCodeEval) and the GitHub repository claims to include experimental data. Raw data is independently verifiable."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.C describes dataset selection (six commonly used APR benchmarks), processing (single-hunk extraction, 8:1:1 split, 1% sampling), and Table IV provides statistics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a purely computational study using pre-existing benchmark datasets. No participant recruitment involved."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: benchmark datasets to single-hunk extraction to 8:1:1 split to 1% sampling (for large datasets) using three seeds to averaging. Table IV shows final counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This research is supported by the Ministry of Education, Singapore under its Academic Research Fund Tier 3 (Award ID: MOET32020-0004).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are affiliated with Singapore Management University, clearly stated on the title page. No commercial affiliations are present."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the Ministry of Education, Singapore — a government academic funding body with no financial interest in prompt tuning vs. fine-tuning outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or declaration of financial interests is present in the paper. Absence of disclosure is not the same as absence of conflict per the schema."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for CodeT5+ or GPT-Neo. These models were trained on web-scraped code data, and the cutoff dates are relevant for assessing benchmark contamination."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between the pre-training data of CodeT5+ or GPT-Neo and the APR benchmark test sets. Public benchmarks like Defects4J could plausibly be in training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmarks like Defects4J (2014) and Code Refinement (2019) were published well before CodeT5+ and GPT-Neo training cutoffs, making contamination plausible. The paper does not address this risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a computational study evaluating LLMs on APR benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Purely computational study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Purely computational study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Purely computational study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Purely computational study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Purely computational study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Purely computational study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, tokens consumed, or cost-per-example reported. Hardware is mentioned (A40 and H100 GPUs) but no cost or timing data."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (NVIDIA A40 48GB and H100 80GB) but total GPU hours, training time, or total compute budget are not reported despite extensive experimentation."
       }
     }
   }
 }