ai-research-survey

scan.json (30290B)

---

 {
   "paper": {
     "title": "Ambig-SWE: Interactive Agents to Overcome Underspecificity in Software Engineering",
     "authors": [
       "Sanidhya Vijayvargiya",
       "Xuhui Zhou",
       "Akhila Yerukola",
       "Maarten Sap",
       "Graham Neubig"
     ],
     "year": 2025,
     "venue": "ICLR 2026",
     "arxiv_id": "2502.13069"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/sani903/InteractiveSWEAgents (footnote 1, page 1). The reproducibility statement also confirms: 'We have also attached the code with the steps to reproduce and the experimental data.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is built on the publicly available SWE-Bench Verified dataset (500 issues). The paper states 'We have also attached the code with the steps to reproduce and the experimental data' in the Reproducibility Statement. Additionally, 'LLM annotations for underspecification are provided in the supplementary materials' (footnote 2)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using OpenHands agent framework (v0.60 for Qwen 3 Coder and Claude Sonnet 4, Appendix A.7), Docker containers from SWE-Bench, and 16 workers in Remote Runtime (beta). However, no requirements.txt, Dockerfile, or detailed dependency specification is provided in the paper text itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The Reproducibility Statement states: 'All key components of the proposed framework are described with the intention of enabling replication by an independent research group. The experimental setup is detailed in §2 and full prompts are provided in the Appendix §A. We have also attached the code with the steps to reproduce and the experimental data.'"
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports resolve rates as point estimates (e.g., percentages in Figure 3, Table 1, Table 2) without confidence intervals or error bars. No ± notation or CI notation is present."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Wilcoxon Signed-Rank Tests (described in Appendix A.3.1) with a significance level of 0.05 to compare Hidden vs. Interaction and Interaction vs. Full settings. Table 4 reports p-values for all six models across both comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context. For example, 'up to 74% over the non-interactive settings' (abstract), and relative performance recovery percentages: 'Claude Sonnet 3.5 models and Haiku 3.5 recover up to 80% of the performance in the Full setting' (§3.2). Resolve rates are reported across all settings (Figure 3) enabling effect magnitude assessment."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses all 500 issues from SWE-Bench Verified for most models, but Claude Sonnet 4 is evaluated on only 100/500 instances in the Hidden setting due to cost (footnote 4). No power analysis is provided, and the choice of 500 issues is inherited from SWE-Bench Verified without independent justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as single-run resolve rates. No standard deviation, variance across runs, or repeated experiment results are provided. The only std dev reported is for the summarization quality metrics (Table 3: ROUGE/BERTScore), not for the main experimental results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes three settings as comparisons: Full (fully specified, no interaction), Hidden (underspecified, no interaction), and Interaction (underspecified, with interaction). The Hidden setting serves as the baseline for measuring interaction benefits, and the Full setting serves as the upper bound."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include contemporary models: Claude Sonnet 4 (2025), Qwen 3 Coder 480B, Claude Sonnet 3.5, Claude Haiku 3.5, Deepseek-v2, and Llama 3.1 70B. These represent current state-of-the-art proprietary and open-weight models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates across multiple dimensions: (1) three experimental settings (Full, Hidden, Interaction) isolating the effect of information and interaction; (2) analysis of navigational vs. informational details (Table 1); (3) three prompt encouragement levels for detection (Table 2); (4) question quality decomposition (§5). These systematically isolate individual factors."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: resolve rate (primary), accuracy/FPR/FNR for detection (Table 2), cosine distance for information gain (Figure 5), LLM-as-judge scores (Figure 6), average number of questions (Table 6), and ROUGE/BERTScore for summarization quality (Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses automated evaluation throughout: test suite pass/fail for resolve rates, LLM-as-judge (GPT-4o) for question quality scoring, and embedding-based cosine distance for information gain. No human evaluation of agent outputs, question quality, or interaction behavior is included. Given that claims are made about question quality and interaction effectiveness, human evaluation would strengthen the findings."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses SWE-Bench Verified's existing test cases to verify solutions. The test cases are separate from the issues provided to agents. There is no tuning/dev split concern since the paper evaluates models without fine-tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides per-model breakdowns across all settings (Figure 3), per-model detection performance across three prompt types (Table 2), per-model analysis of navigational vs. informational information usage (Table 1), and qualitative per-model question strategy analysis (Table 7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple failure patterns: Qwen 3 Coder's complete failure to interact (100% FNR, §4.2), Llama 3.1's poor question quality and vague phrasing (§5.2-5.3), Deepseek's counterintuitive degradation with stronger prompting (§4.2), and Qwen 3 Coder's rigid protocol-following that worsens performance with navigational information (§3.3, Table 1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: interaction does not yield efficiency gains (§3.2), Qwen 3 Coder's resolve rate worsens with navigational information (Table 1), stronger prompting degrades Deepseek's detection (Table 2), and 'prompt engineering offers limited improvement' for detection (§4.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: 'up to 74% over non-interactive settings' is supported by Figure 3 data; 'models struggle to distinguish between well-specified and underspecified instructions' is supported by Table 2; 'significant improvements in performance' is supported by the Wilcoxon tests in Table 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The study design uses controlled comparisons: the same 500 issues are evaluated across Full, Hidden, and Interaction settings with only the information/interaction variable changed. This controlled single-variable manipulation supports causal claims about interaction improving performance. The authors also acknowledge the simulated user proxy limitation and note they 'did not evaluate on naturally underspecified SWE-Bench examples because they lack the paired ground truth necessary for causal measurement of interaction impact' (§2.1)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Interactive Agents to Overcome Underspecificity in Software Engineering' is broader than what was tested. The study is limited to Python repositories from SWE-Bench Verified, synthetic underspecification generated by GPT-4o, and a simulated user proxy (GPT-4o). While the conclusion mentions 'While we focus on software engineering, the methods and insights can extend to other complex, real-world agentic tasks,' the paper does not sufficiently bound its generalizations to the Python/SWE-Bench setting in the title or abstract."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses several alternative explanations: data leakage as a potential factor for Qwen 3 Coder's Hidden performance (§3.2), model's 'superior programming acumen' vs. interaction ability (§3.2), instruction-following capability vs. interaction capability for Sonnet models (§4.3), and rigid protocol-following as an alternative explanation for Qwen 3 Coder's behavior (§3.3)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names without specific API versions or snapshot dates: 'Claude Sonnet 3.5', 'Claude Sonnet 4', 'Claude Haiku 3.5', 'GPT-4o' (for user proxy and summarization), 'Llama 3.1 70B-Instruct', 'Deepseek-v2', 'Qwen 3 Coder 480B'. No API snapshot dates or version identifiers (e.g., 'claude-3-5-sonnet-20241022') are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompts are provided in the Appendix: Full Setting prompt (A.2.1), Interaction Setting prompt with mandatory interaction (A.2.2), User Proxy prompt (A.2.2), Hidden Setting prompt (A.2.3), Summarization prompt (A.2.3), LLM Underspecification Analysis prompt (A.2.3), and three detection prompts (A.5). These are complete prompt texts, not just descriptions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling hyperparameters are reported for any of the models used (coding agents, user proxy GPT-4o, summarization GPT-4o, or embedding model). Only the max turn counts are mentioned (30 for most models, 100 for Claude Sonnet 4 and Qwen 3 Coder)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The OpenHands agentic framework is described in detail (§2.2 and Appendix A.1): tools available (bash terminal, file system, code execution, browsing disabled during evaluation), execution environment (Docker container), agent capabilities (edit files, execute scripts, debug), and completion mechanism (FinishAction, git_patch extraction). The interaction mechanism with user proxy is also fully described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data creation pipeline is documented: starting from 500 SWE-Bench Verified issues, GPT-4o generates underspecified summaries using a provided prompt (A.2.3), the quality is validated with distributional difference analysis and quantitative metrics (Table 3, ROUGE/BERTScore), and LLM annotations of missing information are provided in supplementary materials."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is titled 'Conclusion, Limitations, and Future Work' and contains substantive discussion of limitations spanning an entire paragraph before the findings summary."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: 'Underspecificity detection is measured only within the first three turns, as models rarely recover if they fail to engage early' (§7), 'Question quality is approximated via latent vector changes that weigh all information equally, though models may prioritize details differently' (§7), and 'our simulated user proxy may be more cooperative than real users' (§7)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions 'While we focus on software engineering' in the conclusion, it does not explicitly state what the results do NOT show. It does not bound its findings to Python repositories, SWE-Bench-style tasks, or synthetic underspecification. The conclusion instead extends claims: 'the methods and insights can extend to other complex, real-world agentic tasks.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The reproducibility statement indicates 'We have also attached the code with the steps to reproduce and the experimental data.' The underlying SWE-Bench Verified dataset is publicly available. LLM annotations are provided in supplementary materials."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is well described: SWE-Bench Verified provides the 500 base issues (§2.1), GPT-4o generates underspecified summaries using a specified prompt (§A.2.3), and the quality is validated with distributional difference analysis and quantitative metrics (Table 3)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data source is the standard SWE-Bench Verified benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: SWE-Bench Verified (500 issues) → GPT-4o summarization → distributional difference validation → three experimental settings (Full, Hidden, Interaction) → agent execution in OpenHands Docker containers → git_patch extraction → test verification. The process is clearly traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section listing grants or sponsors is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are disclosed as being from 'Language Technologies Institute, Carnegie Mellon University, Pittsburgh, USA.' The paper evaluates models from Anthropic, Meta, DeepSeek, Alibaba, and OpenAI. CMU is an academic institution with no direct product stake in the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be verified. The paper evaluates commercial products (Claude, GPT-4o, Llama, Deepseek, Qwen) without disclosing whether any company provided funding, credits, or API access."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Graham Neubig is known to be involved with OpenHands/All Hands AI, which is the framework used in this evaluation, but this is not disclosed in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models. This is relevant because SWE-Bench issues are from public GitHub repositories, and models trained after their publication could have seen the solutions."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges potential data leakage: 'Some models achieve higher resolve rates in the Hidden setting likely due to their superior programming acumen, or data leakage' (§3.2). For Qwen 3 Coder specifically, the paper notes 'correct assumptions potentially inflate its performance in this setting' (§3.2, Appendix A.7) and discusses the model's use of internal RFC knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While data leakage is mentioned as a possibility (§3.2), the paper does not systematically address benchmark contamination. SWE-Bench has been publicly available since 2023, and most evaluated models were trained after this date. No temporal analysis, contamination checks, or canary strings are used."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. All evaluations use LLM agents and automated metrics."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-instance costs are reported. The paper mentions cost concerns only qualitatively: 'The model compensates for the lack of information with increased exploration and solution attempts leading to substantially higher evaluation costs' (footnote 4) but does not quantify them."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using '16 workers in the Remote Runtime (beta) provided in OpenHands' (Appendix A.3.2) but does not state total API spend, GPU hours, or total computational budget for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Interactivity can boost performance on underspecified inputs by up to 74% over non-interactive settings.",
       "evidence": "Figure 3 shows resolve rates across Hidden, Interaction, and Full settings for all six models. The 74% figure appears to come from the relative improvement for specific models (e.g., Claude Haiku 3.5 or similar). Wilcoxon tests in Table 4 confirm statistical significance for all models.",
       "supported": "strong"
     },
     {
       "claim": "Claude Sonnet 4 achieves the highest relative performance recovery (89%) in the Interaction setting compared to the Full setting.",
       "evidence": "Stated in §3.2 with resolve rate data from Figure 3. Claude Sonnet 4 demonstrates the strongest ability to integrate information from interaction.",
       "supported": "strong"
     },
     {
       "claim": "Models default to non-interactive behavior unless explicitly prompted, and even with strong encouragement, most models struggle to distinguish well-specified from underspecified inputs.",
       "evidence": "Table 2 shows detection accuracy, FPR, and FNR across three prompt levels for all models. Qwen 3 Coder has 100% FNR across all prompts. Only Claude Sonnet 4 with strong encouragement achieves 89% accuracy.",
       "supported": "strong"
     },
     {
       "claim": "Qwen 3 Coder completely fails to interact under any prompting condition (100% FNR across all prompts).",
       "evidence": "Table 2 shows FNR = 1.00 for Qwen 3 Coder across Neutral, Moderate, and Strong encouragement conditions. Discussed in §4.2.",
       "supported": "strong"
     },
     {
       "claim": "Claude Sonnet 4 achieves comparable information gain to Qwen 3 Coder (0.171 vs 0.179 cosine distance) with 50% fewer questions (4.03 vs 6.02).",
       "evidence": "Figure 5 shows cosine distance values, and Table 6 shows average question counts. Discussed in §5.2.",
       "supported": "strong"
     },
     {
       "claim": "Interaction improves task success but does not yield efficiency gains (no reduction in action steps).",
       "evidence": "§3.2 states that Qwen 3 Coder uses roughly the same number of steps (65) in both settings, while Claude Sonnet 4 increases from 65 to 75 steps with interaction.",
       "supported": "moderate"
     },
     {
       "claim": "As model capability scales (e.g., from Claude Sonnet 3.5 to Claude Sonnet 4), substantial performance gains occur in both interactive and non-interactive settings.",
       "evidence": "Figure 3 data comparing resolve rates across Claude model generations. §3.2 discusses this scaling trend.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper introduces Ambig-SWE, a benchmark for evaluating LLM agents' ability to handle underspecified software engineering tasks through interaction. Across six models, interaction recovers significant performance lost to underspecification (up to 74% improvement), with Claude Sonnet 4 achieving the best recovery rate (89% of full-specification performance). Most models default to non-interactive behavior and struggle to detect underspecification, with Qwen 3 Coder completely failing to interact under any prompting condition. Effective interaction strategies prioritize exploration-first approaches and targeted questions over high question volume.",
   "red_flags": [
     {
       "flag": "Synthetic underspecification may not represent real-world patterns",
       "detail": "Underspecified issues are generated by GPT-4o, not collected from actual user interactions. The paper acknowledges that natural underspecified issues differ (having more concrete technical details, reproducibility information, external references, and conversational fragments). The synthetic approach uses 'more aggressive information removal' which may not match real user behavior."
     },
     {
       "flag": "Simulated user proxy instead of real users",
       "detail": "GPT-4o serves as the user proxy, which may be more cooperative, consistent, and predictable than real users. The paper acknowledges this limitation but does not quantify the gap. Real users might provide noisy, incomplete, or misleading responses."
     },
     {
       "flag": "No confidence intervals or variance across runs",
       "detail": "All resolve rates are reported as single-run point estimates without error bars, confidence intervals, or repeated runs. LLM outputs are stochastic, so single-run results may not be stable. Only p-values from Wilcoxon tests are provided."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any model. These significantly affect LLM output behavior and reproducibility."
     },
     {
       "flag": "Unequal evaluation conditions across models",
       "detail": "Claude Sonnet 4 was evaluated on only 100/500 instances in the Hidden setting due to cost (footnote 4), while all other models were evaluated on all 500. Claude Sonnet 4 and Qwen 3 Coder received 100 turns vs. 30 for other models. Qwen 3 Coder received a modified prompt with mandatory clarification phase. These asymmetries complicate cross-model comparison."
     },
     {
       "flag": "Undisclosed potential conflict of interest",
       "detail": "Graham Neubig (senior author) is associated with OpenHands/All Hands AI, the agent framework used for all experiments. This relationship is not disclosed in the paper, though it could influence framework selection and configuration choices."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Foundational benchmark for evaluating LLM agents on real-world software engineering tasks, directly used as the base dataset in this work."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "The agentic framework used for all experiments in this paper, evaluating LLM capabilities in software engineering tasks."
     },
     {
       "title": "The impact of AI on developer productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Empirical study on AI-assisted programming productivity, relevant to understanding the real-world impact of code generation tools."
     },
     {
       "title": "Generative AI at work",
       "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey R Raymond"],
       "year": 2023,
       "relevance": "Foundational study on productivity effects of generative AI in workplace settings."
     },
     {
       "title": "TheAgentCompany: Benchmarking LLM agents on consequential real world tasks",
       "authors": ["Frank F. Xu", "Yufan Song", "Boxuan Li"],
       "year": 2024,
       "arxiv_id": "2412.14161",
       "relevance": "Benchmark for evaluating LLM agents on real-world tasks beyond code generation, including interactive scenarios."
     },
     {
       "title": "AgentCoder: Multi-agent-based code generation with iterative testing and optimisation",
       "authors": ["Dong Huang", "Jie M. Zhang", "Michael Luck"],
       "year": 2024,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent approach to code generation with testing feedback, relevant to agentic software engineering evaluation."
     },
     {
       "title": "ClarifyGPT: Empowering LLM-based code generation with intention clarification",
       "authors": ["Fangwen Mu", "Lin Shi", "Song Wang"],
       "year": 2023,
       "arxiv_id": "2310.10996",
       "relevance": "Directly addresses clarification in LLM code generation, a core topic of this paper's evaluation."
     },
     {
       "title": "LLM-based test-driven interactive code generation: User study and empirical evaluation",
       "authors": ["Sarah Fakhoury", "Aaditya Naik", "Georgios Sakkas", "Saikat Chakraborty", "Shuvendu K. Lahiri"],
       "year": 2024,
       "doi": "10.1109/tse.2024.3428972",
       "relevance": "Empirical study of interactive test-driven code generation with LLMs including user study, relevant to understanding human-AI interaction in coding."
     },
     {
       "title": "The AI Scientist: Towards fully automated open-ended scientific discovery",
       "authors": ["Chris Lu", "Cong Lu", "Robert Tjarko Lange", "Jakob Foerster", "Jeff Clune", "David Ha"],
       "year": 2024,
       "arxiv_id": "2408.06292",
       "relevance": "Agentic AI system for scientific research, relevant to understanding capabilities and limitations of autonomous AI agents."
     },
     {
       "title": "Aligning language models to explicitly handle ambiguity",
       "authors": ["Hyuhng Joon Kim", "Youna Kim", "Cheonbok Park"],
       "year": 2024,
       "arxiv_id": "2404.11972",
       "relevance": "Addresses LLM handling of ambiguous inputs, directly related to the underspecificity challenges studied in this paper."
     },
     {
       "title": "Sotopia: Interactive evaluation for social intelligence in language agents",
       "authors": ["Xuhui Zhou", "Hao Zhu", "Leena Mathur"],
       "year": 2024,
       "arxiv_id": "2310.11667",
       "relevance": "Framework for evaluating interactive capabilities of language agents in social settings, related to the interaction evaluation methodology."
     },
     {
       "title": "Learning to ask: When LLMs meet unclear instruction",
       "authors": ["Wenxuan Wang", "Juluan Shi", "Chaozheng Wang"],
       "year": 2024,
       "arxiv_id": "2409.00557",
       "relevance": "Evaluates LLM behavior on ambiguous tool-use instructions, directly relevant to understanding how models handle underspecification."
     }
   ]
 }