ai-research-survey

scan.json (19216B)

---

 {
   "paper": {
     "title": "Responsible Artificial Intelligence for Earth Observation: Achievable and realistic paths to serve the collective good",
     "authors": ["Pedram Ghamisi", "Weikang Yu", "Andrea Marinoni", "Caroline M. Gevaert", "Claudio Persello", "Sivasakthy Selvakumaran", "Manuela Girotto", "Benjamin P. Horton", "Philippe Rufin", "Patrick Hostert", "Fabio Pacifici", "Peter M. Atkinson"],
     "year": 2024,
     "venue": "arXiv preprint / IEEE MGRS",
     "arxiv_id": "2405.20868",
     "doi": "10.1109/MGRS.2025.3529726"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis", "qualitative"],
   "key_findings": "This paper surveys responsible AI practices in Earth observation across five dimensions: mitigating unfair bias, AI security (adversarial defense, uncertainty, XAI), geo-privacy, ethical principles (scientific excellence, open data), and AI4EO for social good. It identifies specific challenges like spatial autocorrelation bias unique to geospatial data, privacy concerns with high-resolution UAV/satellite imagery, and the need for stakeholder co-creation. The paper provides two concrete application examples (early warning systems for mass movements, climate teleconnections) but presents no new empirical evaluation or systematic quantitative analysis.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code or analysis scripts are released. No repository URL provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset or corpus of reviewed papers is released. The Google Scholar search described in Fig. 1 is not provided as structured data."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/review paper with no computational experiments requiring an environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No instructions for reproducing the literature search or analysis. The Google Scholar keyword search in Fig. 1 is loosely described but not reproducible."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no statistical experiments. Fig. 1 shows publication counts but no statistical analysis."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical comparisons are made in this survey."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments producing effect sizes."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experimental sample involved."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs to report variance across."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not compare itself against prior surveys of responsible AI in EO or position itself relative to specific prior reviews in a structured way."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baseline comparison applies since no structured comparison with prior surveys is attempted."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No evaluation metrics used — this is a narrative review."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No test set involved."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper organizes its review into clear thematic categories (bias, security, privacy, ethics, social good) with dedicated sections for each."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure modes and challenges extensively: spatial autocorrelation biases (Section 2.3.2), adversarial attacks (Section 3.1.1), domain shift issues (Section 3.1.2), privacy violations from UAV imagery (Section 4.1), and stigmatization risks (Section 5.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports limitations of existing approaches: adversarial training only defends against specific attacks (Section 3.2.1), current XAI methods produce semantically misaligned visualizations (Section 3.3.3), and upscaling studies are lacking (Section 5.5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to 'systematically define the intersection of AI and EO with a central focus on responsible AI practices' and identify 'critical components.' The paper delivers on this with dedicated sections for each component (bias, security, privacy, ethics, social good)."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey that does not make causal claims about its own work. It describes causal relationships from the literature (e.g., spatial autocorrelation causing overestimation) but attributes these to cited sources."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to be a 'pioneering effort to systematically define the intersection of AI and EO' but does not bound this claim — it does not specify which EO subfields, AI methods, or geographic contexts are covered vs excluded. The literature coverage is not described with explicit inclusion/exclusion criteria."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/taxonomy paper presenting no empirical results that would require alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "No measurements or proxies — this is a narrative review."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No models are used in this survey."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting used."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a Google Scholar search for Fig. 1 ('keywords: machine learning or deep learning and remote sensing') but provides no systematic review protocol, no inclusion/exclusion criteria for the papers discussed, and no documentation of how the reviewed literature was selected."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section 8 (Conclusions) acknowledges geopolitical complexities but does not discuss limitations of the review itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to the validity of the review are discussed. The paper does not address potential selection bias in its literature coverage, language bias, or coverage gaps."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims broad coverage ('pioneering and thorough review') but does not state what is explicitly out of scope, which EO subfields or AI methods are excluded, or what geographic/temporal bounds apply to the literature reviewed."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data from the literature search or analysis is available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions a Google Scholar search for publication counts (Fig. 1) but does not describe how the broader body of reviewed literature was collected or selected."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is published literature, which is a standard corpus, though the selection process is undocumented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No pipeline from literature search to review synthesis is documented. The paper jumps from a general framing to thematic discussion without describing how papers were found, screened, or organized."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed, including Maxar Technologies (a commercial satellite provider) for Fabio Pacifici."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. One author is from Maxar Technologies, a commercial EO provider with a potential interest in the paper's conclusions about responsible AI in EO."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. Maxar Technologies is mentioned favorably in Section 7 (SpaceNet, ESG reports) by an author affiliated with Maxar."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no pre-trained model evaluated on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no benchmark evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no benchmark evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no method with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper — no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No structured review protocol is followed. No PRISMA diagram, no registered protocol, no systematic search strategy with reproducible queries. The paper is a narrative review with ad-hoc paper selection."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of the papers it cites. All sources are treated equally regardless of their rigor."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias in the reviewed literature. The survey does not consider whether its sources skew toward positive results about AI4EO."
       }
     }
   },
   "claims": [
     {
       "claim": "AI4EO publications in IEEE TGRS have surged dramatically since 2020, growing from ~500 in 2019 to ~2500 in 2023.",
       "evidence": "Fig. 1 shows publication counts from Google Scholar advanced search using keywords 'machine learning' or 'deep learning' and 'remote sensing' in IEEE TGRS from 2014-2023.",
       "supported": "moderate"
     },
     {
       "claim": "Spatial autocorrelation between training and testing samples causes overestimation of model accuracy in geospatial applications.",
       "evidence": "Cites Karasiak et al. (2022) [28] who demonstrated this effect. Section 2.3.2.",
       "supported": "strong"
     },
     {
       "claim": "Graph-based ensemble neural network approaches can deliver robust hindcasting and forecasting of mass movement impacts at local, regional, and national scales.",
       "evidence": "Section 6.1 cites Dimasaka et al. (2023) [142] showing results over 68,000 incidents since 1957 across Norwegian territory.",
       "supported": "weak"
     },
     {
       "claim": "The field has transitioned from singular focus on model-centric or data-centric approaches to a balanced data-model-centric paradigm.",
       "evidence": "Fig. 2 and Section 1 present this as the authors' opinion ('we believe'), supported by the rise of foundation models. No systematic evidence is provided.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No systematic review methodology",
       "detail": "The paper claims to be a 'pioneering and thorough review' but follows no structured review protocol (PRISMA, etc.). Paper selection appears ad-hoc with no documented search strategy, inclusion/exclusion criteria, or screening process."
     },
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Author Fabio Pacifici is affiliated with Maxar Technologies. Section 7 mentions Maxar favorably multiple times (SpaceNet co-founder, ESG reports). No competing interests statement addresses this."
     },
     {
       "flag": "No quality assessment of reviewed sources",
       "detail": "The survey does not assess the methodological quality of cited works. All sources are treated as equally reliable, potentially laundering weak results alongside strong ones."
     },
     {
       "flag": "Claims outrun evidence",
       "detail": "The paper calls itself 'a pioneering effort to systematically define the intersection of AI and EO' but it is a narrative review, not a systematic one. The word 'systematically' is not justified by the methodology."
     }
   ],
   "cited_papers": [
     {
       "title": "Gender Shades: Intersectional Accuracy Disparities in Commercial Gender Classification",
       "authors": ["Joy Buolamwini", "Timnit Gebru"],
       "year": 2018,
       "relevance": "Foundational work on AI bias in facial recognition, relevant to AI fairness and responsible AI evaluation."
     },
     {
       "title": "A framework for understanding sources of harm throughout the machine learning life cycle",
       "authors": ["Harini Suresh", "John Guttag"],
       "year": 2021,
       "relevance": "Framework for understanding ML bias types (historical, representation, measurement, etc.) relevant to AI methodology quality."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
       "year": 2023,
       "arxiv_id": "2304.15004",
       "relevance": "Questions whether LLM emergent abilities are artifacts of evaluation metrics — directly relevant to AI evaluation methodology."
     },
     {
       "title": "On the Opportunities and Risks of Foundation Models",
       "authors": ["Rishi Bommasani"],
       "year": 2021,
       "relevance": "Comprehensive analysis of foundation model risks and opportunities, relevant to AI safety and responsible deployment."
     },
     {
       "title": "Universal adversarial examples in remote sensing: Methodology and benchmark",
       "authors": ["Yonghao Xu", "Pedram Ghamisi"],
       "year": 2022,
       "relevance": "Benchmark for adversarial robustness in remote sensing AI, relevant to AI security evaluation methodology."
     },
     {
       "title": "Tackling climate change with machine learning",
       "authors": ["David Rolnick"],
       "year": 2022,
       "relevance": "Survey of ML applications for climate change, relevant to AI for social good evaluation."
     },
     {
       "title": "Large Language Models are Geographically Biased",
       "authors": ["Rohin Manvi", "Samar Khanna", "Marshall Burke", "David Lobell", "Stefano Ermon"],
       "year": 2024,
       "relevance": "Documents geographic bias in LLMs, directly relevant to AI bias and fairness research methodology."
     },
     {
       "title": "Deep learning in remote sensing: A comprehensive review and list of resources",
       "authors": ["Xiao Xiang Zhu"],
       "year": 2017,
       "relevance": "Major survey of deep learning in remote sensing, relevant as a methodological comparison for AI survey quality."
     }
   ]
 }