ai-research-survey

scan.json (24203B)

---

 {
   "paper": {
     "title": "Bridging Human Interpretation and Machine Representation: A Landscape of Qualitative Data Analysis in the LLM Era",
     "authors": ["Xinyu Pi", "Qisen Yang", "Chuong Nguyen", "Hua Shen"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.11739",
     "doi": "10.48550/arXiv.2601.11739"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL is provided in the paper. The paper describes annotation prompts in the appendix but does not link to a repository or code archive."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions releasing 'the full rubric, prompts, and boundary heuristics used for annotation in the Appendix G' but the annotated paper dataset, the 300 annotated items, and the curated corpus are not released as downloadable data. The appendix contains prompts but not the raw annotation data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or software versions are provided. The paper mentions using GPT-5.2 and Qwen3-32B but provides no setup details for reproducing the computational components."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes its methodology conceptually and includes annotation prompts in the appendix, but there are no concrete instructions for reproducing the annotation study or the operationalizability experiment."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports exact-match agreement rates (85% for meaning-making, 94% for modeling) from the GPT-5.2 operationalizability experiment but provides no confidence intervals or error bars for these figures."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make comparative claims that would require significance testing. It presents a conceptual framework and descriptive distributions, not performance comparisons between competing methods."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make comparative claims requiring effect sizes. Its empirical component is descriptive annotation, not an experiment comparing treatments or methods."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 231 empirical QR papers (randomly sampled from 32,321 eligible candidates) and 69 computational papers but does not justify these sample sizes with a power analysis or explicit rationale for why these numbers are sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "The study does not involve multiple experimental runs or repeated measurements. The annotation study produces a single consensus label per paper, so variance across runs is not applicable."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a novel conceptual framework paper. There is no prior competing landscape or taxonomy of qualitative analysis levels that would serve as a baseline. The contribution is the framework itself."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable as no baselines are expected for a novel conceptual framework paper."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The framework is a conceptual 4x4 landscape, not a system with removable components. Ablation is structurally inapplicable."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The operationalizability experiment reports exact-match agreement on both meaning-making (85%) and modeling (94%) axes separately, providing two distinct evaluation dimensions (Sec. 3.4, App. H)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The core annotation study (Sec. 4) involves manual human annotation by three authors who independently annotate each paper on both axes, with disagreements resolved through discussion. This constitutes human evaluation of the framework's operationalizability."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a predictive modeling study. There is no train/test split paradigm applicable to a conceptual framework annotation study."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides a detailed breakdown of paper distributions across all 16 cells of the 4x4 landscape, separately for empirical QR papers and computational papers. The observations (O1, O2, O3) discuss patterns across specific regions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses where the landscape reveals gaps in LLM-based QR (observations O1-O3 in Sec. 4), including that computational work clusters in modeling-shallow regions (D1-D2) and the gap between human QR practice and automated approaches."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core finding is itself a negative result: existing LLM-based qualitative analysis systems are skewed toward low-level meaning-making and low-commitment modeling, failing to match the depth of human QR practice. The 85% agreement on meaning-making (vs 94% on modeling) also shows imperfect operationalizability."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a 'strong skew toward low-level meaning and low-commitment representations' which is supported by the annotation distribution in Figure 2 and observations O1-O3 in Section 4. The claim about the 4x4 landscape is structural and fully described in the paper."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make causal claims. It presents a descriptive framework and documents the distribution of existing work across the landscape. Language like 'reveals' and 'highlights' is descriptive, not causal."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Limitations and Risks) explicitly states: 'the empirical distributions reported in this paper are derived from a curated sample of qualitative and computational works rather than from an exhaustive or statistically representative census of the field' and 'the observed skew...should be interpreted as descriptive of our sampled corpus, not as a population-level estimate.' This is careful bounding."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses that the observed skew could be partly an artifact of query-based retrieval and venue visibility, noting the corpus 'may systematically under-represent certain disciplines, methodological traditions, languages, or forms of qualitative practice.' This acknowledges sampling-based alternative explanations for the observed distributions."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'GPT-5.2' for the operationalizability experiment (Sec. 3.4) and 'Qwen3-32B' for the eligibility triage (Sec. 4). While these are marketing names without API snapshot dates, the paper provides specific named versions rather than just 'GPT' or 'Qwen.'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts for the annotation tasks are provided in Appendix G, including the paper classification prompt (G.1), the level of meaning-making prompt (G.2), and the level of modeling prompt (G.3). These are complete prompt texts, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for the GPT-5.2 or Qwen3-32B runs (temperature, top-p, max tokens, etc.). The paper does not mention any sampling or generation settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The LLM calls (GPT-5.2, Qwen3-32B) are single-shot classification tasks, not multi-step agentic workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents the PRISMA-style pipeline for empirical QR papers with specific counts at each stage: 664,244 records identified, 138,683 after dedup/screening, 32,321 after eligibility triage, 231 randomly sampled for annotation. The filtering criteria at each stage are described (keyword combinations, downloadability, LLM triage for empirical QR content)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is titled 'Limitations and Risks' and contains a substantial multi-paragraph discussion of both conceptual limitations (LLMs lack lived experience, positionality) and methodological limitations (sampling bias, reproducibility concerns, automation bias risks)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats including: (1) LLMs may produce 'fluent but ungrounded interpretations' especially for M3-M4 level tasks, (2) the corpus is conditioned on keyword-based retrieval which may 'systematically under-represent certain disciplines, methodological traditions, languages,' (3) reproducibility concerns from sensitivity to 'prompting, sampling, context windowing, and model/version changes,' and (4) automation bias risks. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states the annotation study goal 'is not to exhaustively review the QR literature, but to test whether our two axes...are operational, discriminative, and faithful' (Sec. 4). Section 7 states distributions 'should be interpreted as descriptive of our sampled corpus, not as a population-level estimate.' The computational paper collection is acknowledged to be 'conditioned on query terms and visibility.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw annotation data (per-annotator labels, disagreement logs, the full list of 300 annotated papers with their assigned levels) are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the collection process in detail: for empirical QR papers, 4,493 domain keywords crossed with 10 QR methodology terms yielding 44,930 query combinations searched on Google Scholar. For computational papers, snowball sampling from seed queries. Both processes are described with stages and counts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for a study. The annotators were the three co-authors themselves. This is a paper annotation study, not a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented with counts at each stage: identification (664,244), screening/dedup (138,683), eligibility triage via Qwen3-32B (32,321), and random sampling (231). For computational papers, the snowball sampling process is described yielding 69 papers. The annotation protocol is also described with three annotators, independent dual annotation, and consensus resolution."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. There is no mention of grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: three authors at UC San Diego and one at NYU Shanghai / New York University. The paper does not evaluate any specific LLM product from these institutions, so there is no obvious conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. The absence of a funding disclosure is itself a gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. The GPT-5.2 operationalizability experiment tests whether the framework definitions are clear enough for an independent agent to apply, not the model's inherent knowledge. Contamination is not a concern here."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model capabilities is performed. The GPT-5.2 pass is a framework operationalizability test, not a model capability evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model capabilities is performed. Contamination is structurally inapplicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The annotation study uses the three co-authors as annotators on research papers, not human subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Annotating published research papers does not require IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The annotators are the paper's authors."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in an experimental study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in an experimental study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper is primarily a conceptual framework paper. The LLM usage (GPT-5.2, Qwen3-32B) is ancillary to the main contribution. Cost reporting is not a meaningful expectation for a framework/taxonomy paper."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is primarily a theoretical framework paper with a manual annotation study. Computational resources are minimal and ancillary."
       }
     }
   },
   "claims": [
     {
       "claim": "Existing LLM-based qualitative analysis systems exhibit a strong skew toward low-level meaning-making (M1-M2) and low-commitment modeling (D1-D2), with few reliable attempts at interpretive/theoretical inference or dynamical modeling.",
       "evidence": "Figure 2 shows the annotated distribution of 69 computational papers concentrated in D1-D2 on the modeling axis, with most at M1-M2 on meaning-making. Observation O1 (Sec. 4) documents this pattern explicitly.",
       "supported": "moderate"
     },
     {
       "claim": "Human empirical QR papers are meaning-deep but often representation-light, occupying M3-M4 on meaning-making but typically only D1-D3 on modeling.",
       "evidence": "Observation O2 (Sec. 4) based on the annotation of 231 empirical QR papers shown in Figure 2, demonstrating right-shifted distribution on meaning-making axis but not consistently high on modeling axis.",
       "supported": "moderate"
     },
     {
       "claim": "In high-end human QR, meaning and modeling are coupled: high-level modeling (D3-D4) almost never appears with low-level meaning-making (M1-M2).",
       "evidence": "Observation O3 (Sec. 4) notes the upper-left region of Figure 2 (high modeling, low meaning) is essentially empty for empirical papers.",
       "supported": "moderate"
     },
     {
       "claim": "The 4x4 landscape framework is sufficiently operational for independent annotation, achieving 85% exact-match agreement on meaning-making and 94% on modeling when applied by GPT-5.2.",
       "evidence": "Section 3.4 reports GPT-5.2 annotation pass using only definitions without access to human annotations, compared against human consensus labels (detailed in App. H).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["theoretical", "qualitative", "observational"],
   "key_findings": "The paper introduces a 4x4 landscape for characterizing qualitative research outputs, crossing four levels of meaning-making (descriptive, categorical, interpretive, theoretical) with four levels of modeling (static, stages, causal, dynamics). An annotation study of 300 papers (231 empirical QR + 69 computational) reveals that LLM-based systems concentrate heavily in shallow meaning-making and low-commitment modeling regions, while human qualitative research achieves deeper meaning but often without corresponding representational richness. The framework achieves 85-94% exact-match agreement when applied by GPT-5.2, suggesting reasonable operationalizability.",
   "red_flags": [
     {
       "flag": "Self-annotation bias",
       "detail": "All three annotators are co-authors of the paper who also developed the framework being validated. This creates a potential confirmation bias since they have deep familiarity with the intended distinctions and may apply them more consistently than independent annotators would."
     },
     {
       "flag": "No inter-rater reliability statistics",
       "detail": "While the paper mentions independent dual annotation with consensus resolution through discussion, no inter-rater reliability statistics (e.g., Cohen's kappa, Fleiss' kappa, Krippendorff's alpha) are reported for the human annotation. Only the GPT-5.2 vs. human agreement is reported."
     },
     {
       "flag": "Computational paper sample not systematic",
       "detail": "The 69 computational papers were curated through snowball sampling rather than systematic retrieval, and the paper acknowledges this corpus 'is conditioned on query terms and visibility.' The distributional claims about computational QR may be biased by the non-systematic sampling."
     },
     {
       "flag": "GPT-5.2 validation is circular",
       "detail": "Using GPT-5.2 to validate the operationalizability of a framework for LLM-assisted qualitative analysis raises concerns. The high agreement could reflect that LLMs are good at following structured rubrics rather than that the framework captures meaningful distinctions in practice."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Multi-agent LLM failure analysis using qualitative methods, directly relevant to agentic AI evaluation methodology."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Foundational multi-agent LLM framework, relevant to agentic AI systems surveyed in the project."
     },
     {
       "title": "MetaGPT: Meta Programming for a Multi-Agent Collaborative Framework",
       "authors": ["Sirui Hong"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent collaborative framework for software development, relevant to agentic AI programming."
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian", "Wei Liu"],
       "year": 2024,
       "relevance": "Multi-agent system for software development, directly relevant to agentic AI in code generation."
     },
     {
       "title": "Professional software developers don't vibe, they control: AI agent use for coding in 2025",
       "authors": ["Ruanqianqian Huang"],
       "year": 2025,
       "arxiv_id": "2512.14012",
       "relevance": "Study of how professional developers use AI coding agents, relevant to AI programming productivity research."
     },
     {
       "title": "Large Language Models in Qualitative Research: Uses, Tensions, and Intentions",
       "authors": ["Hope Schroeder"],
       "year": 2025,
       "relevance": "Examines LLM use in qualitative research contexts, relevant to AI-assisted research methodology."
     },
     {
       "title": "Measuring agents in production",
       "authors": ["Melissa Z Pan"],
       "year": 2025,
       "arxiv_id": "2512.04123",
       "relevance": "Measurement methodology for deployed AI agents, relevant to agentic AI evaluation."
     },
     {
       "title": "Grounded Copilot: How Programmers Interact with Code-Generating Models",
       "authors": ["Shraddha Barke"],
       "year": 2022,
       "relevance": "Qualitative study of programmer interactions with code generation models, directly relevant to AI programming research."
     },
     {
       "title": "Large language model for qualitative research – a systematic mapping study",
       "authors": ["Cauã Ferreira Barros"],
       "year": 2025,
       "arxiv_id": "2411.14473",
       "relevance": "Systematic mapping of LLM use in qualitative research, relevant to understanding AI research methodology."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM technical report, relevant to understanding model capabilities underlying agentic AI systems."
     }
   ]
 }