ai-research-survey

calibration.json (18228B)

---

 {
   "paper_slug": "across-programming-language-2025",
   "total_questions": 50,
   "agreement_count": 48,
   "disagreement_count": 2,
   "agreement_rate": 0.96,
   "disagreements": [
     {
       "category": "statistical_methodology",
       "question": "effect_sizes_reported",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "opus_justification": "The schema states that 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper's tables consistently show both baseline and enhanced absolute Pass@K values alongside percentage improvements (e.g., Table II: baseline 55.28 → 95.85, '73% up'). This provides the magnitude of effect with full baseline context. While no formal effect size metric like Cohen's d is used, the schema explicitly allows 'percentage improvement with baseline context' as sufficient.",
       "sonnet_justification": "The paper reports percentage improvements (e.g., '73% improvement', '+15.75% for multi-lingual LLMs') but without any standardized effect sizes (Cohen's d, odds ratios) or baseline context sufficient for formal effect size assessment. While relative improvements are shown, no formal effect size metric is used.",
       "direction": "opus_generous"
     },
     {
       "category": "conflicts_of_interest",
       "question": "funder_independent_of_outcome",
       "sonnet_applies": false,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": false,
       "opus_justification": "The schema says 'NA if unfunded.' Sonnet marked applies=false reasoning that no funding is disclosed making it unassessable. However, the authors are at major academic institutions (Chinese Academy of Sciences, HKUST) where research is typically funded. The absence of a funding disclosure does not mean the work is unfunded — it means we cannot verify funder independence. Since it is not clearly unfunded solo independent work, applies=true, answer=false is more appropriate.",
       "sonnet_justification": "No funding is disclosed, making this criterion not assessable. Authors are from academic institutions with no apparent financial stake in any evaluated model.",
       "direction": "applies_boundary"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section X (Data Availability) provides a URL: https://anonymous.4open.science/r/Cross-Lingual-RACG-0F3C. The paper states 'We released the artifact and all experiment data' at this link."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The 13,910-instance Multilingual Code Dataset Expansion is released at the artifact URL (Section X). HumanEval-X is a publicly available dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions specific model names and greedy decoding settings (Section II.E), but provides no requirements.txt, Dockerfile, or library version list. Insufficient detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references the artifact URL but provides no step-by-step reproduction instructions within the paper text. No README commands or scripted replication pipeline described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Only point estimates (Pass@K values) reported across all tables. The Std columns represent cross-language spread, not uncertainty over experimental runs. No confidence intervals or error bars provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Comparative claims (e.g., 'Java demonstrates superior cross-lingual utility over Python') are made throughout by comparing point estimates without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables consistently show absolute baseline and enhanced Pass@K values alongside percentage improvements (e.g., Table II: baseline 55.28 mean → 95.85 mean, '73% up'). Per the schema, 'percentage improvement with baseline context' is sufficient for YES. Both raw differences and relative improvements are provided with full context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains approximately 1K instances per language (Section II.C) but no power analysis or explicit justification for this sample size. The size was determined by data availability, not principled calculation."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Experiments use greedy decoding (temperature=0.0, Section II.E) producing single deterministic outputs. No variance across runs is reported. The Std columns in tables represent cross-language variation, not across-run variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "All experiments compare against 'Baseline (without RAG/injection)' throughout Tables II-IX. Retrieval strategies are compared against each other in Table X (BM25, BGE-large, CodeRankEmbed)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models from 2023-2024 (CodeLlama, DeepSeek-Coder, Qwen2.5-Coder). CodeRankEmbed is from 2024. These are contemporary at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Doc vs Doc w/o NL settings (removing NL comments), knowledge injection vs full RACG pipeline, and mono-lingual vs multi-lingual LLMs comparisons systematically isolate individual components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Pass@K for generation evaluation (Section II.D), Precision@K and Recall@K for retrieval evaluation (Table X). Multiple metrics used across different experimental aspects."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation using automated test case execution (Pass@K). Human evaluation is clearly irrelevant when executable unit tests provide definitive correctness judgments."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses test case execution on generated code, which is independent of the retrieval corpus. Test cases are held out from the retrieval and generation process. The evaluation mechanism (pass/fail on unit tests) is separate from retrieved documents."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "All result tables provide per-programming-language breakdowns across all 13 PLs (Tables III, V, VI, IX). Language-specific performance variations are explicitly analyzed."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure modes: PHP and Scala corpus degrading performance (Table V), Python mono-lingual LLMs suffering native language degradation, Perl and Go corpus causing widespread degradation for mono-lingual LLMs. Section VIII also acknowledges limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results reported: cross-lingual RACG degrading mono-lingual LLMs on native Python (-4.57% in Table V), some corpora (PHP, Scala, Go) causing net-negative effects, cross-lingual knowledge sometimes interfering with existing strengths."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The four key abstract insights (effectiveness, inequality, robustness, specialization) are each supported by corresponding experimental results in Tables II-X. Abstract claims do not exceed what the results show."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The controlled knowledge injection setting (Section II.B.1) isolates the generation variable using an oracle retriever. Adversarial attacks (RQ3) directly manipulate corpus documents while holding other variables constant. These controlled manipulations are adequate for the causal inferences drawn."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only 7B-parameter models but the conclusion claims to 'establish foundational insights for designing more powerful and safer code intelligence.' Section VIII acknowledges model size as a threat but abstract and conclusion still make broad, unbounded claims about multi-lingual RACG systems generally."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper asserts singular interpretations for findings without systematically considering alternatives. For example, the claim that 'multi-lingual LLMs rely on internal knowledge to resist adversarial perturbations' is not tested against alternatives (e.g., distributional shift, tokenization differences). Section VIII discusses experimental limitations, not alternative explanations for observed results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section II.E specifies exact model identifiers: CodeLlama-7B-Instruct, Deepseek-Coder-6.7B-Instruct, Qwen2.5-Coder-7B-Instruct, Phi-1, Phi-1.5, and CodeRankEmbed. These are specific HuggingFace model names mapping to unique checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section II.E states 'we use a unified prompt template for LLMs, following the design in [19] and [43]' but the actual prompt text is not provided in the paper or appendix. Readers must consult referenced papers to reconstruct prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section II.E reports temperature=0.0 (greedy decoding), top-K=3 retrieval window, and fixed random seed of 42 for perturbation experiments. Key hyperparameters are stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The RACG pipeline is described in Section II: retriever (CodeRankEmbed) retrieves top-K documents, injected into prompt for LLM generation. Figure 1 illustrates the full pipeline with four experimental settings. The scaffold is sufficiently described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II.C documents dataset construction: starting from HumanEval-X and extensions of [43][44], LLM-generated reference solutions via Qwen2-72B-Instruct-GPTQ-Int4, 5 iterations of generation + unit test verification, per-language counts documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII 'Threats to Validity' is a dedicated section discussing methodological limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII identifies two specific threats: (1) models not exceeding 7B parameters may not represent broader LLM landscape, and (2) fixed random seed (42) limits perturbation diversity. Both are specific to this study with mitigation measures described."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Section VIII hedges about model representativeness but does not state specific untested claims. The conclusion broadly claims to 'establish foundational insights for designing more powerful and safer code intelligence' without bounding these claims to the tested setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The artifact URL (Section X) includes 'all experiment data' including the 13,910-instance dataset. Raw data is available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section II.C describes dataset construction: source datasets (HumanEval-X, [43], [44]), LLM solution generation, unit test verification across 5 iterations, per-language instance counts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sourced from existing benchmarks (HumanEval-X, McEval, MultiPL-E) and LLM-generated solutions verified by unit tests."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II.C documents the full pipeline: source datasets → LLM generation of missing solutions → unit test verification over 5 iterations → 13,910 final instances with per-language counts (Python: 1181, Kotlin: 1071, etc.)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure appears in the paper. No mention of grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Chinese Information Processing Laboratory (Institute of Software, CAS), University of Chinese Academy of Sciences, and Hong Kong University of Science and Technology."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. Authors are at major academic institutions (CAS, HKUST) where research is typically funded. Since funding is not disclosed, funder independence cannot be verified. This is not clearly unfunded solo independent work, so the criterion applies but is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement appears in the paper. Absence of disclosure is not absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates models (CodeLlama, DeepSeek-Coder, Qwen2.5-Coder, Phi-1, Phi-1.5) on HumanEval-X without stating any model's training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval-X is derived from HumanEval (published 2021). All evaluated models were trained on data post-2021 and may have seen HumanEval problems. No discussion of potential train/test overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021; all tested models (CodeLlama-2023, DeepSeek-Coder-2024, Qwen2.5-Coder-2024, Phi-2023) have training cutoffs after its release. The paper does not address whether HumanEval problems appeared in training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLM-based code generation systems."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not an experimental study with human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Blinding not applicable to automated benchmark evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, tokens consumed, or wall-clock time reported. Given the scale of experiments (13,910 instances across multiple models and settings), this is a notable omission."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total compute budget stated despite using Qwen2-72B for dataset generation and multiple 7B models for evaluation."
       }
     }
   }
 }