ai-research-survey

scan.json (21307B)

---

 {
   "paper": {
     "title": "AI Safety Subproblems for Software Engineering Researchers",
     "authors": ["David Gros", "Prem Devanbu", "Zhou Yu"],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2304.14597"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides GitHub links for the keyword matching regex (https://github.com/DNGros/aisse/blob/main/ai_terms_regex.txt) and the list of foundational safety papers (https://github.com/DNGros/aisse/blob/main/foundation_papers.csv)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The foundational papers CSV is released via GitHub. The underlying citation data comes from the publicly available Semantic Scholar corpus, which is referenced. The analysis inputs are thus reproducible from public sources."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency information are provided. The paper references using the Semantic Scholar API and corpus but does not specify software versions or dependencies needed to reproduce the analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the methodology at a high level (querying Semantic Scholar API, filtering by venue) but does not provide runnable scripts or detailed steps to reproduce the citation counts in Table 1."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The citation counts in Table 1 are presented as point estimates with no confidence intervals or error bars. The paper acknowledges limitations of the proxy measure but provides no quantification of uncertainty."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make comparative claims that would require significance tests. It presents descriptive counts of citations rather than claiming statistical differences between groups."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper is primarily descriptive and theoretical. The citation counts serve as a motivating observation, not as the basis for effect size claims."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 44 foundational safety works (FSafe) is described as informed by querying references in survey papers, but no formal justification is given for this number or the completeness of the set. The paper acknowledges limitations but does not justify the sample."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "There are no experimental runs or repeated measurements. The citation analysis is a single deterministic query against the Semantic Scholar corpus."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a position/perspective paper with a supporting literature analysis, not an evaluation of a system or method. There is no system to compare against baselines."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no system or method is being evaluated against baselines."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate. This is a position paper proposing research directions."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No system evaluation is performed. The citation count analysis is descriptive, not an evaluation metric."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs to evaluate. This is a theoretical/position paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No ML model or system is evaluated on test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides a per-venue breakdown of CSafe paper counts across SE, ML, AI, NLP, CV, PL, and other conference/journal categories, rather than just an overall aggregate."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses an 'anti-problem' (Section 3.5) — research directions that might seem safety-related but are not net-beneficial. It also extensively discusses limitations of its citation analysis in Section 1."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main finding IS a negative result: only 4 papers at major SE venues reference foundational AI safety work, and 0 at PL venues. The anti-problems section (3.5) also presents research that is not beneficial to safety."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 'quantified literature review suggesting that AI Safety discussions are not common at SE venues.' Table 1 supports this with only 4 CSafe papers at SE conferences and 5 at SE journals. The abstract also claims to 'categorize subproblems' which Section 3 delivers."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It presents descriptive observations and proposes future research directions. Language is appropriately hedged with terms like 'conjectures' and 'suggest.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states limitations: 'This analysis has many limitations. Large amounts of research (like in AI robustness or interpretability) is safety-motivated, even if it never references the FSafe discussions of long-term AI.' The analysis is clearly bounded to citation-based proxy measures."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations for the low citation counts: safety-motivated research may not cite the specific foundational works selected, and 'AI Safety culturally relies heavily on self-publishing or web forums' which the analysis would miss. Section 2.1 discusses common skepticisms about AI safety."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No AI/ML models are used in the paper's own methodology. The citation analysis uses the Semantic Scholar API, not a language model."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. The analysis is based on citation graph queries, not LLM prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No model training or inference is performed. The citation analysis has no hyperparameters in the ML sense."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a citation analysis and position paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the methodology at a high level — querying Semantic Scholar, filtering CSafe to 2012-March 2023, filtering by 'Computer Science' field — but does not document specific preprocessing steps, API query parameters, or how venue classifications were resolved (e.g., footnote 7 notes a misclassification issue). The regex for AI/ML/DL terms is released but the full pipeline steps are not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Limitations are discussed inline in Section 1 ('This analysis has many limitations...') but there is no substantive standalone section addressing limitations of the overall paper or its proposals."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: safety-motivated research that doesn't cite FSafe works would be missed, the reliance on Semantic Scholar corpus classifications (footnotes 7-8 note misclassifications), the cultural tendency of AI safety to rely on self-publishing, and that 33% of SE papers mentioning AI is based on keyword matching which may over- or under-count."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'This set of conjectures and sub-problems is not intended to be comprehensive or definitive. Instead it is a set of example starting points. Solving these problems alone will not solve AI alignment.' It also notes it focuses on technical alignment rather than policy."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The foundational papers CSV is released, but the full dataset of 6565 CSafe works, their citation links, and the venue classification data are not released. An independent researcher cannot verify the counts in Table 1 without re-running the analysis."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes using the Semantic Scholar API and corpus to query citations to 44 foundational works, filtering to 2012-March 2023, and using venue classifications. The FSafe selection process is described as 'informed by querying all references in a set of survey papers and examining works that are in multiple surveys.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data comes from automated citation graph queries on a public corpus."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline is only described at a high level. Key details are missing: how FSafe was filtered from survey references to exactly 44 works, how venue classifications were resolved, what API endpoints were used, and how the 33% estimate of AI-related SE papers was computed. The regex is released but not the full pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgements state: 'DG's research work was partially supported by the NSF (CCF-1934568).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: David Gros and Prem Devanbu at University of California, Davis; Zhou Yu at Columbia University. These are academic institutions with no apparent conflict regarding AI safety advocacy."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is an independent government funding agency with no financial stake in whether AI safety is discussed more at SE venues."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model's capability on a benchmark. It performs a citation graph analysis using the Semantic Scholar API."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark. Contamination is not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of a pre-trained model is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a citation analysis and position paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a position/theoretical paper with a supporting literature analysis. No system with inference costs is proposed or evaluated."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a position paper. The citation analysis uses API queries with negligible compute. No compute budget is relevant."
       }
     }
   },
   "claims": [
     {
       "claim": "AI Safety discussions are not common at SE venues: only 4 papers at major SE conferences (ICSE, FSE, ASE, ISSTA) out of 7,744 reference foundational AI safety works.",
       "evidence": "Table 1 shows 4 CSafe papers at SE conferences out of 7,744 total, and 5 at SE journals out of 6,082. This is based on citation graph analysis using Semantic Scholar corpus (Section 1).",
       "supported": "moderate"
     },
     {
       "claim": "Approximately 33% of papers at large SE venues in 2022 mentioned AI/ML/DL terms in their title or abstract, compared to 4% in 2012.",
       "evidence": "Section 1, based on keyword matching in the Semantic Scholar corpus. The regex used is released on GitHub. However, the methodology could over- or under-count.",
       "supported": "moderate"
     },
     {
       "claim": "AI safety topics are discussed most often at ML-focused conferences (NeurIPS, ICLR, ICML, KDD) with 225 papers, and AI-specific conferences (AAAI, IJCAI) with 87 papers.",
       "evidence": "Table 1 provides venue-by-venue counts from the citation graph analysis.",
       "supported": "moderate"
     },
     {
       "claim": "There are 12 concrete AI safety subproblems that fit into traditional SE research areas.",
       "evidence": "Section 3 enumerates 12 problems (P1-P12) across 4 conjectures. These are proposed research directions, not empirically validated claims.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["theoretical", "qualitative"],
   "key_findings": "This 4-page position paper argues that AI safety is under-discussed at SE venues, supported by a citation graph analysis showing only 4 papers at major SE conferences reference foundational AI safety works (out of 7,744 total). The authors propose 12 concrete subproblems organized under 4 conjectures about how software engineering will change with rising AI capabilities (machine-written code, increased testing needs, universal software engineering, AI in critical systems). They also identify an 'anti-problem': improving code generation correctness metrics does not necessarily contribute to long-term safety.",
   "red_flags": [
     {
       "flag": "Proxy measure validity unclear",
       "detail": "The citation-based proxy for 'considering AI safety' (citing any of 44 foundational works) may significantly undercount safety-relevant research. The paper acknowledges this but the magnitude of undercounting is unknown. Research on robustness, interpretability, or fairness may be safety-motivated without citing the specific FSafe papers."
     },
     {
       "flag": "Selection of foundational works not fully transparent",
       "detail": "The 44 FSafe works were selected by 'examining works that are in multiple surveys,' but the threshold (how many surveys?) and exact selection criteria are not documented. Different selections could yield substantially different counts."
     },
     {
       "flag": "Subproblems are speculative conjectures",
       "detail": "The 12 proposed subproblems (P1-P12) are framed as conjectures without empirical validation. The paper acknowledges this ('not intended to be comprehensive or definitive') but readers should note these are research direction proposals, not findings."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "relevance": "Introduces HumanEval benchmark for code generation, which is discussed in the paper as an example of capability metrics that are not safety objectives."
     },
     {
       "title": "Taking Flight with Copilot: Early Insights and Opportunities of AI-Powered Pair-Programming Tools",
       "authors": ["Christian Bird"],
       "year": 2023,
       "relevance": "Early empirical insights on AI-powered pair programming (GitHub Copilot), directly relevant to AI4SE evaluation."
     },
     {
       "title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback",
       "authors": ["Yuntao Bai"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Foundational RLHF alignment work relevant to LLM safety methodology."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI Feedback",
       "authors": ["Yuntao Bai"],
       "year": 2022,
       "relevance": "Key alignment technique (Constitutional AI) for language model safety."
     },
     {
       "title": "Unsolved Problems in ML Safety",
       "authors": ["Dan Hendrycks"],
       "year": 2021,
       "arxiv_id": "2109.13916",
       "relevance": "Comprehensive categorization of ML safety problems, directly relevant to surveying AI safety methodology."
     },
     {
       "title": "Large Language models and Simple, Stupid, Bugs",
       "authors": ["Kevin Jesse"],
       "year": 2023,
       "relevance": "Studies how LLM code generators produce buggy code, relevant to AI code generation quality evaluation."
     },
     {
       "title": "Language Models (Mostly) Know What They Know",
       "authors": ["Saurav Kadavath"],
       "year": 2022,
       "relevance": "Studies calibration and uncertainty in language models, relevant to evaluating LLM self-knowledge and safety."
     },
     {
       "title": "How to certify machine learning based safety-critical systems? A systematic literature review",
       "authors": ["Florian Tambon"],
       "year": 2021,
       "relevance": "SLR on ML safety certification in SE context, directly relevant to AI safety methodology quality."
     },
     {
       "title": "Software Engineering for AI-Based Systems: A Survey",
       "authors": ["Silverio Martínez-Fernández"],
       "year": 2021,
       "relevance": "Major SE4AI survey published in TOSEM, relevant to understanding SE approaches to AI system quality."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "InstructGPT paper on RLHF for instruction following, foundational to LLM alignment evaluation."
     },
     {
       "title": "Ethical and social risks of harm from Language Models",
       "authors": ["Laura Weidinger"],
       "year": 2021,
       "arxiv_id": "2112.04359",
       "relevance": "Taxonomy of LLM risks and harms, relevant to evaluating AI safety research methodology."
     }
   ]
 }