ai-research-survey

scan.json (20999B)

---

 {
   "paper": {
     "title": "Collaborating with GenAI: Incentives and Replacements",
     "authors": ["Boaz Taitler", "Omer Ben-Porat"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2508.20213",
     "doi": "10.48550/arXiv.2508.20213"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or link is provided in the paper. The simulations described in Section 6 are not accompanied by released code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The simulation data (10,000 generated instances) is not released. No data download links or supplementary data files are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware ('intel Core i7-9700k CPU and 16 GB RAM') in Section 6 but provides no software environment details, library versions, or dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The simulation setup is described at a high level in Section 6 (instance generation procedure, best-response dynamics with tolerance 10^-8), but there are no step-by-step instructions or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The simulation results in Section 6 report empirical frequencies and distributions over 10,000 instances but provide no confidence intervals or error bars on these estimates."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper is primarily theoretical with proofs. The simulations illustrate theoretical findings rather than making comparative statistical claims that would require significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents theoretical results (propositions, theorems) and illustrative simulations. Effect sizes in the statistical sense are not applicable to formal proofs or frequency distributions of game-theoretic outcomes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The simulation uses 10,000 generated instances (Section 6) but provides no justification for why this number was chosen or whether it is sufficient for the claims being illustrated."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Simulation results are presented as aggregate empirical frequency distributions (Figures 1 and 2) without reporting variance, standard deviation, or spread measures across runs or subsets."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper introducing a new game-theoretic model. There is no prior model of the same problem to serve as a baseline comparison. The paper compares GenAI vs. no-GenAI settings within its own framework."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable for the same reason as baselines_included — this is a novel theoretical framework, not a benchmark evaluation."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a theoretical model with proofs. There is no system with components to ablate. The comparison of GenAI vs. no-GenAI settings (Section 3) is part of the core analysis, not an ablation."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper proving properties of a game-theoretic model. Evaluation metrics in the empirical sense do not apply."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper with mathematical proofs and simulations. Human evaluation of outputs is not relevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No datasets or test sets are used. The paper presents theoretical analysis and simulations of a game-theoretic model."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1b breaks down optimal coalition size by the total player share P, providing a granular view across different parameter ranges rather than just overall aggregates."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explicitly discusses instability of optimal coalitions (Proposition 5.2), showing cases where the optimal coalition is not stable. The myopic removal dynamics analysis (Section 6.1) shows ~80% of instances degrade to the empty coalition."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: GenAI can lead to zero effort even when nearly ineffective (Proposition 3.6), optimal coalitions are often unstable (Proposition 5.2), and myopic removal dynamics typically lead to empty coalitions (~80% of instances, Section 6.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported by formal proofs: GenAI leading to no effort (Proposition 3.5, 3.6), NP-completeness of the optimization problem (Theorem 4.2), efficient algorithm for linear cases (Theorem 4.4), low-value workers being critical (Proposition 5.3), and cascade effects (Section 5.3 and simulations in Section 6)."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes formal mathematical claims about equilibrium properties of a game-theoretic model, not empirical causal claims. The 'causes' are within the model's assumptions, established through proofs."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit about its theoretical nature and limitations. Section 7 states 'Our paper focuses on a theoretical perspective that is naturally limited' and discusses specific assumptions (exogenous portions, free GenAI access). The model's scope is clearly bounded to the MSB game framework."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper presenting mathematical proofs within a formal model. Alternative explanations in the empirical sense do not apply — the results follow logically from the model's axioms."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No LLM or AI model is used in the experiments. The paper is a theoretical game-theory study with numerical simulations."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. This is a theoretical paper with mathematical simulations, not an LLM-based study."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports simulation hyperparameters: 12 players per instance, coefficients sampled uniformly from [0,1], tolerance of 10^-8 for best-response dynamics, 10,000 instances generated. The shared benefit function form (product) and contribution/cost function forms are fully specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a theoretical game-theory paper."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 fully describes the instance generation procedure: how contribution functions, cost functions, shared benefit functions, and player portions are constructed from sampled parameters. The pipeline from parameter sampling to equilibrium computation is documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 ('Discussion and Future Work') discusses limitations including the assumption of exogenous portions, the lack of contract design analysis, and the theoretical nature of the work ('Our paper focuses on a theoretical perspective that is naturally limited')."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 identifies specific limitations: exogenously given portions (theta_i) which may not hold in all scenarios, the assumption that GenAI is free, and the lack of diverse interaction forms (competition, substitutable relationships). These are specific to this study's design choices."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly identifies what was NOT studied: varying portions to reach better equilibria, contract design where players must outperform GenAI, and competition or substitutable relationships. The paper also states in Section 1.1 that it 'focuses on a theoretical perspective that is naturally limited.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 10,000 simulation instances and their results are not made available for independent verification. Only aggregate figures are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 ('Generating instances') describes in detail how simulation data was generated: 12 players, product shared benefit function, contribution functions with sqrt(alpha_i*e_i + beta_i*g_i), linear costs, and parameter sampling from U[0,1]."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical paper with computational simulations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 documents the full pipeline: instance generation (parameter sampling) → brute-force coalition search over all 2^12 subsets → best-response dynamics with 10^-8 tolerance for each coalition → aggregation into frequency distributions. Hardware and total runtime (15 hours) are also reported."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section states: 'This research was supported by the Israel Science Foundation (ISF; Grant No. 3079/24).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are clearly affiliated with Technion—Israel Institute of Technology, with email addresses provided on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Israel Science Foundation is a public research funding agency with no commercial stake in the results of this game-theoretic study about GenAI incentives."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model's capability on a benchmark. It is a theoretical game-theory paper with numerical simulations."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on any benchmark. Not applicable."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. Not applicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a theoretical paper with computational simulations."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper. There is no proposed method with inference costs. The simulations are illustrative, not a deployable system."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports: 'We used a standard PC with intel Core i7-9700k CPU and 16 GB RAM for running the simulations. The entire execution took roughly 15 hours.'"
       }
     }
   },
   "claims": [
     {
       "claim": "GenAI can lead workers to exert no effort in equilibrium, even when GenAI's effectiveness is negligible.",
       "evidence": "Proposition 3.6 (Section 3.2) proves that for any epsilon > 0, there exists a game where GenAI changes contribution by at most epsilon but the dominant equilibrium shifts from full effort (e_i=1) to zero effort (e_i=0). Proof uses a constructed 2-player game instance.",
       "supported": "strong"
     },
     {
       "claim": "The manager's problem of finding the optimal coalition is NP-complete.",
       "evidence": "Theorem 4.2 (Section 4.1) with full proof in Appendix B.1, using a reduction from the clique problem. The construction maps vertices to players and edges to shared benefit terms.",
       "supported": "strong"
     },
     {
       "claim": "An efficient O(N/epsilon^2) algorithm exists for the special case of (almost-)linear shared benefit functions.",
       "evidence": "Theorem 4.4 (Section 4.2) with Algorithm 1 in Appendix B.2. The reduction to a knapsack problem is shown, and correctness/runtime are formally proven.",
       "supported": "strong"
     },
     {
       "claim": "Workers with near-zero marginal contribution can be pivotal for sustaining overall output; their removal triggers cascades.",
       "evidence": "Proposition 5.3 (Section 5.2) proves for any epsilon > 0, there exists an instance where a player's marginal contribution is less than epsilon yet is in the optimal coalition. Section 5.3 and Proposition 5.4 show decreasing VSR values trigger cascading exclusions.",
       "supported": "strong"
     },
     {
       "claim": "Optimal coalitions tend to be either large or small (rarely medium-sized), and ~80% degrade to empty coalitions under myopic removal dynamics.",
       "evidence": "Simulations over 10,000 instances (Section 6.1, Figures 1 and 2). Coalitions of 10-12 players appear ~70% of the time, medium coalitions (1-8) are rare. Under myopic removal dynamics, ~80% converge to empty coalitions.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["theoretical"],
   "key_findings": "This paper presents a game-theoretic model (Managed Shared Benefit game) showing that GenAI access can paradoxically reduce worker effort to zero in equilibrium, even when GenAI provides negligible performance improvement. The manager's optimal team selection problem is NP-complete in general, with an efficient algorithm for linear cases. Simulations over 10,000 instances demonstrate that optimal coalitions are bimodal (large or empty) and highly unstable under myopic removal dynamics, with cascading exclusions triggered by removing even low-contribution workers.",
   "red_flags": [
     {
       "flag": "Simulations only illustrate theoretical claims",
       "detail": "The simulations in Section 6 use a specific parametric form (multiplicative shared benefit, specific contribution/cost functions with uniformly sampled coefficients). The results may not generalize beyond this narrow functional class, though the paper does not claim they do."
     },
     {
       "flag": "No code or data released for simulation verification",
       "detail": "The 10,000-instance simulation study is not reproducible without access to code. The specific instance generation, best-response dynamics implementation, and convergence criteria cannot be independently verified."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the impact of early-2025 AI on experienced open-source developer productivity",
       "authors": ["J. Becker", "N. Rush", "E. Barnes", "D. Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "Empirical study of AI's impact on developer productivity, directly relevant to the survey's core topic of AI-augmented software engineering."
     },
     {
       "title": "The impact of AI on developer productivity: Evidence from GitHub Copilot",
       "authors": ["S. Peng", "E. Kalliamvakou", "P. Cihon", "M. Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Landmark study on AI-assisted programming productivity, directly relevant to evaluating claims about GenAI's effect on developer output."
     },
     {
       "title": "Strategic foundation models",
       "authors": ["D. Goktas", "A. Greenwald", "T. Osogami", "R. Patel", "K. Leyton-Brown"],
       "year": 2025,
       "relevance": "Examines strategic aspects of foundation models including delegation and deferring to AI, relevant to understanding AI agent safety and alignment."
     },
     {
       "title": "Social choice for AI alignment: Dealing with diverse human feedback",
       "authors": ["V. Conitzer", "R. Freedman", "J. Heitzig"],
       "year": 2024,
       "doi": "10.48550/ARXIV.2404.10271",
       "relevance": "Addresses AI alignment through social choice theory, relevant to the survey's coverage of AI safety and alignment research."
     },
     {
       "title": "Braess's paradox of generative AI",
       "authors": ["B. Taitler", "O. Ben-Porat"],
       "year": 2025,
       "relevance": "Game-theoretic analysis showing GenAI can paradoxically reduce outcomes, directly relevant to understanding unintended consequences of AI adoption."
     },
     {
       "title": "Competition and diversity in generative AI",
       "authors": ["M. Raghavan"],
       "year": 2024,
       "arxiv_id": "2412.08610",
       "relevance": "Examines competitive dynamics of generative AI systems, relevant to understanding market and ecological effects of AI tools."
     },
     {
       "title": "Fine-tuning games: Bargaining and adaptation for general-purpose models",
       "authors": ["B. Laufer", "J. Kleinberg", "H. Heidari"],
       "year": 2024,
       "relevance": "Game-theoretic framework for fine-tuning foundation models, relevant to understanding strategic interactions around AI model development."
     },
     {
       "title": "A no free lunch theorem for human-AI collaboration",
       "authors": ["K. Peng", "N. Garg", "J. Kleinberg"],
       "year": 2025,
       "relevance": "Theoretical result on fundamental limits of human-AI collaboration, directly relevant to the survey's assessment of AI-augmented work claims."
     }
   ]
 }