ai-research-survey

scan.json (18884B)

---

 {
   "paper": {
     "title": "Autonomous Supplier Evaluation and Data Stewardship with AI: Building Transparent and Resilient Supply Chains",
     "authors": ["Chandra Bonthu", "Ganpati Goel"],
     "year": 2025,
     "venue": "International Journal of Computational and Experimental Science and Engineering (IJCESEN)",
     "doi": "10.22399/ijcesen.3854"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or any mention of code release found in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a manufacturing-level schema with multiple data sources but does not release any dataset or provide download links."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'pinned containers' and 'locked versions of an operating system, compiler, and CUDA/cuDNN' conceptually but does not provide actual version numbers, requirements files, or environment specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses metrics like PR-AUC, F1@k, NDCG, Brier score, and ECE but does not report any confidence intervals or error bars on these results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the full stack 'surpasses baselines' but provides no statistical significance tests. Comparisons are presented as qualitative findings in Table 2 without p-values or formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 describes findings qualitatively (e.g., 'Recall drops', 'Ranking degrades') without reporting actual numerical effect sizes, percentage improvements, or magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not disclose the number of suppliers, purchase orders, or observations in the dataset, let alone justify the sample size."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across runs, or spread measures are reported for any experimental results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 states 'Baselines included a weighted scorecard, logistic regression on tabular features, and a gradient-boosted tree trained without text or graph input.'"
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are heuristic scorecards, logistic regression, and GBT without text/graph — these are reasonable but no contemporary ML-based supplier evaluation methods from the literature are compared against."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents ablations removing text features, graph features, comparing model families (tree ensembles vs tabular deep models), and comparing calibration methods (Platt vs isotonic)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses PR-AUC, F1@k, NDCG@5/10, Brier score, and ECE as evaluation metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes human-in-the-loop triage and manual audit of text-dominated cases in Section 5.4, but these are part of the system design, not a systematic human evaluation of system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes a rolling-origin procedure with sequential training, validation, and test windows, and supplier blocking to ensure unseen suppliers in test folds."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions delivery, quality, and compliance risk categories and multiple horizons (30/60/90 days), but Table 2 does not provide per-category numerical breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 discusses false positives (episodic expedite spikes not reflecting continuity risk) and false negatives (aggressive caches obscuring near-miss delays), with qualitative error analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The calibration ablation reports that Platt scaling yields 'higher variance than isotonic', and the paper acknowledges tree ensembles outperform deep tabular models, suggesting the latter was a less successful approach."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'Experiments show that it is better for early warning of risks' and 'calibrated ranking strategies are more effective than static thresholds,' but the results in Table 2 are entirely qualitative with no numerical values to verify these claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies systematically remove individual modalities (text, graph) to measure their contribution, constituting controlled single-variable manipulation adequate for the causal claims made about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 explicitly bounds generalizability: 'Category dynamics constrain generalizability: semiconductor lead-time shocks contrast with packaging disruptions.' The paper acknowledges results are contingent on governance quality and organizational maturity."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 discusses survivorship bias, coverage gaps at tier-n levels, labeling distortion from reporting delays, and measurement error in OCR-extracted certificates as alternative explanations for observed performance."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'gradient-boosted tree', 'transformer-based models', 'tabular deep models' without specifying exact model names, versions, or library versions."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use LLM prompting. It uses traditional ML models (gradient-boosted trees, transformer encoders for text embedding)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 mentions 'Hyperparameters were identified on validation ranges via a multi-objective search' but does not report the actual hyperparameter values used."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a traditional ML pipeline, not an LLM agent."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and 3.2 document data preprocessing in detail: entity resolution, FX normalization, unit dictionaries, UTC synchronization, working-day calendars, SLA harmonization, temporal truncation at label anchor, and data contract enforcement."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 is titled 'Limitations & Threats to Validity' and provides substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 discusses specific threats: survivorship bias from attrited suppliers, tier-n coverage gaps undermining centrality features, labeling distortion from delayed shipment reports, OCR measurement error in certificates, and model-behavior drift from supplier adaptation to inspection routines."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 states that gains are 'contingent upon the quality of governance, the maturity of data-observability, and the organization's ability to take action on recommendations.' Category-specific dynamics (semiconductor vs packaging) are noted as limiting generalizability."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is released. The paper describes the schema (Table 1) but provides no access to the underlying procurement data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Table 1 describe data sources in detail: purchase orders, goods receipt notes, ASN, accounts payable, QMS records, audit findings, TMS milestones, certificate OCR, and external risk feeds."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data comes from enterprise procurement systems, not recruited subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.3 document the pipeline from raw data sources through entity resolution, normalization, feature engineering, labeling with temporal safeguards, and governance controls at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section found in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Chandra Bonthu at EVERSANA (Director MDM) and Ganpati Goel at Zero Motorcycles Inc."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The authors work at companies (EVERSANA, Zero Motorcycles) that could benefit from supplier evaluation systems, but this potential conflict is not discussed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses traditional ML models (gradient-boosted trees, transformer encoders) trained on proprietary data, not pre-trained LLMs evaluated on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on a benchmark. The paper trains its own models on proprietary procurement data with temporal cross-validation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model benchmark evaluation. The paper uses internal procurement data, not public benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in a research study sense. The paper describes a system deployment, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 mentions 'inference latency' as part of multi-objective hyperparameter search and Section 5.3 mentions load testing inference delays, but no actual cost or latency numbers are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, training time, or computational budget is reported despite describing containerized training with CUDA/cuDNN."
       }
     }
   },
   "claims": [
     {
       "claim": "The full stack (tabular + text + graph) surpasses baselines for early risk detection in delivery, quality, and compliance.",
       "evidence": "Table 2 and Section 5.2 state the full stack surpasses baselines but provide only qualitative findings without numerical results.",
       "supported": "weak"
     },
     {
       "claim": "Removing text features (audit/narrative embeddings) causes recall to drop for compliance-related events.",
       "evidence": "Table 2 modality ablation and Section 5.2 describe the ablation qualitatively: 'Removal of unstructured features decreased recall at matched precision of compliance-related events.'",
       "supported": "weak"
     },
     {
       "claim": "Removing graph features degrades ranking for thin-history suppliers.",
       "evidence": "Table 2 and Section 5.2: 'omitting the feature set associated with graphs, the poor rank statistics on the thin-history suppliers were not observed.' No numerical values provided.",
       "supported": "weak"
     },
     {
       "claim": "Tree ensembles show stronger discrimination and resilience to missing data compared to tabular deep models.",
       "evidence": "Table 2 model family ablation, described qualitatively in Section 5.2 without specific metrics.",
       "supported": "weak"
     },
     {
       "claim": "Calibrated ranking strategies are more effective than static thresholds in limited review capacity settings.",
       "evidence": "Abstract and Section 5.2 mention decision-curve analysis but provide no numerical comparisons of capture rates or expected costs.",
       "supported": "weak"
     },
     {
       "claim": "Stewardship practices decrease override rates and improve adoption.",
       "evidence": "Section 5.5 states 'precipitous decreases in rates because calibration had stabilized' and Section 6.5 mentions 'lower override levels' but provides no specific numbers.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "The paper proposes a multi-modal ML pipeline for supplier risk evaluation combining tabular, text (audit narratives), and graph (supply network) features with data governance controls. Ablation studies suggest each modality contributes unique signal, with text features improving compliance recall and graph features improving ranking for low-history suppliers. However, all experimental results are reported only qualitatively in Table 2 without any numerical values, confidence intervals, or statistical tests, making it impossible to assess the magnitude or reliability of claimed improvements.",
   "red_flags": [
     {
       "flag": "No numerical results",
       "detail": "Table 2 (the main results table) contains only qualitative descriptions of findings ('Recall drops', 'Ranking degrades', 'Tree ensembles show stronger discrimination') without a single numerical value. No PR-AUC scores, F1 values, NDCG numbers, or Brier scores are reported despite being listed as evaluation metrics."
     },
     {
       "flag": "Dataset size undisclosed",
       "detail": "The number of suppliers, purchase orders, events, or observations in the dataset is never stated. Without knowing the scale, it is impossible to assess whether the evaluation is meaningful."
     },
     {
       "flag": "No quantitative evidence for any claim",
       "detail": "Every empirical claim in the paper (abstract and Section 5.2) is unsupported by numerical evidence. The paper describes metrics and evaluation procedures in detail but never reports actual measured values."
     },
     {
       "flag": "Potential industry conflict not disclosed",
       "detail": "Authors work at EVERSANA (MDM/data management) and Zero Motorcycles (manufacturing). Both could benefit from supplier evaluation tools. No competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Attention Is All You Need",
       "authors": ["Vaswani, A."],
       "year": 2017,
       "relevance": "Foundational transformer architecture referenced for text encoding of audit narratives in the pipeline."
     },
     {
       "title": "A Dynamic Memory-Based Approach for Natural Language Inference",
       "authors": ["Raju"],
       "year": 2017,
       "relevance": "Referenced for dynamic memory inference approach used in the text encoder to reconcile contradictory audit notes."
     },
     {
       "title": "Dual Sourcing for Supply Chain Resilience",
       "authors": ["Goel", "Bhramhabhatt"],
       "year": 2024,
       "relevance": "Referenced for dual-sourcing strategies as risk mitigation in supply chains, authored by one of the paper's authors."
     }
   ]
 }