ai-research-survey

scan.json (26262B)

---

 {
   "paper": {
     "title": "SocialVeil: Probing Social Intelligence of Language Agents under Communication Barriers",
     "authors": ["Keyang Xuan", "Pengda Wang", "Chongrui Ye", "Haofei Yu", "Tal August", "Jiaxuan You"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.05115"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Communication barriers consistently impair LLM agents' social intelligence, with mutual understanding reduced by over 45% and confusion elevated by nearly 50%. Semantic vagueness most severely disrupts mutual understanding (avg -58%), emotional interference disproportionately damages relationship quality (avg -49%), and sociocultural mismatch induces persistent confusion (avg -49%). Adaptation strategies (repair instruction and interactive learning) yield only modest improvements, with performance remaining far below barrier-free baselines.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our code and data are available at https://github.com/ulab-uiuc/social-veil' in a footnote on page 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code and data are available at the GitHub link. Episodes are adapted from SOTOPIA, which is publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Training details mention '4 × A6000 80G GPUs' and QLoRA parameters (Appendix D.3), but no requirements.txt, Dockerfile, or dependency specifications are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but the paper itself contains no README-style instructions for replicating experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported throughout: barrier identification accuracy (Table 4, Figure 6), ICC values, Pearson correlations, and bootstrap CIs for human evaluation metrics."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Statistical significance is reported: ICC F-tests with p < .001, Pearson correlations with significance levels (*p < .05, **p < .01, ***p < .001) in Figure 4, and bootstrap resampling for stability estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as percentage changes from baseline: 'mutual understanding reduced by over 45%', 'avg −58%' for semantic vagueness on mutual understanding. Figure 5 shows percentage deviations with significance markers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 180 episodes per barrier type were chosen, nor is a power analysis discussed. The 720 total scenarios and 120 human evaluation samples are stated without justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports standard deviations (subscript notation) for all metrics across all models and conditions. For example, 'GPT-4o-m Base BEL 8.78.11' indicates mean 8.78 with std 0.11."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "A barrier-free baseline condition is included for all experiments, allowing direct comparison of performance with and without communication barriers (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include contemporary LLMs: GPT-4o-mini, Qwen2.5-7B-Instruct, Qwen3-4B-Instruct, and Mistral-8B-Instruct. The framework builds on SOTOPIA (2023), which is the current standard."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests three distinct barrier types independently and a barrier-free baseline, effectively ablating the barrier dimension. Table 3 ablates adaptation strategies (Base vs Repair Instruction vs BC+SR)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six evaluation metrics are used: Believability, Relationship Quality, Knowledge, Goal Completion, Unresolved Confusion, and Mutual Understanding (Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Six human annotators evaluated 120 scenarios across barrier identification accuracy, inter-rater reliability (Fleiss's Kappa, ICC), and metric alignment with automated scores (Section 5.2, Table 4)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit separation of dev and test sets is described. The 180 episodes per condition appear to be used both for development and reporting. Interactive learning uses filtered trajectories from the same scenario pool."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by barrier type (semantic, sociocultural, emotional), by model, and by difficulty split (All vs Hard) in Table 2. Figure 5 shows per-barrier-type deviations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses the failure of repair instruction strategies, and Section 5.1 analyzes behavioral patterns of communication breakdown. The paper discusses when and why agents fail to adapt."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that repair instruction yields 'trivial performance improvements' and that neither adaptation strategy restores baseline performance (Section 4.3, Table 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '45% reduction in mutual understanding' and '50% elevation in confusion' are supported by Table 2 data. Human evaluation ICC≈0.78 and Pearson r≈0.80 are confirmed in Section 5.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims (barriers cause performance degradation) are supported by controlled experimental design: the barrier is the sole manipulated variable between conditions, with partner agents held constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds claims to the tested models and the SOTOPIA scenario set. Appendix B explicitly discusses limitations: text-only interactions, short-term scenarios, and the gap to multimodal communication."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper validates barriers are not artifacts of specific models (Appendix F.1 tests alternative evaluator backbone, F.2 tests alternative barrier backbone). Section 5.1 provides linguistic feature analysis as independent validation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly introduces barrier-aware metrics (Unresolved Confusion, Mutual Understanding) alongside goal-oriented metrics, distinguishing between task success and social competence. The evaluation protocol (Appendix G) precisely defines what each metric measures."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix D.2 lists HuggingFace model IDs for open-source models (e.g., 'Qwen/Qwen2.5-7B-Instruct', 'mistralai/Mistral-8B-Instruct') but GPT-4o-mini and GPT-4o lack snapshot dates or API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix H: agent prompt (H.2), barrier evaluation prompt (H.3), social goal evaluation prompt (H.4), and neutralization prompt (H.1)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature 0.7 for agents, 0 for evaluator. Training details: learning rate 5.0e-5, batch size 4, 20 epochs, cutoff length 4096, QLoRA rank 8, alpha 16, dropout 0.05 (Appendix D.3)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agent scaffolding is described in detail: two-layer barrier design with style prompts and parameterization, episode structure with private goals and role profiles, turn-based dialogue with 20-turn cap (Sections 2.2-2.3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The scenario neutralization process is described: GPT-4o rewrites SOTOPIA scenarios to remove goal-related hints (Section 2.3), with the exact neutralization prompt provided (Appendix H.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides a dedicated limitations and future directions section discussing text-only focus, short-term interaction limitation, and episode-level evaluation constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B discusses specific limitations: text-only modality misses prosody/facial expressions, scenarios are discrete/short-term rather than longitudinal, and evaluation protocol operates at episode level rather than capturing proactive barrier mitigation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: focuses on cognitive-factor-induced barriers (not physical), text-only interactions, and three specific barrier types. Appendix B states what the framework does NOT cover."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code and data are available at the GitHub repository, which would include the generated dialogues and evaluation results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Episode generation is described in detail: scenarios adapted from SOTOPIA, neutralized using GPT-4o, barrier injection via style prompts and parameterization, with 180 episodes per condition (Section 2.2-2.3)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "For human evaluation: 'Six human annotators were recruited from two universities and received research credit, identified as 50% women and 50% men' (Appendix E). However, no discussion of whether this introduces selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: SOTOPIA scenarios → neutralization → barrier injection → multi-turn dialogue simulation → automated evaluation + human evaluation. Interactive learning pipeline is also documented (behavior cloning → self-reinforcement)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UIUC (Siebel School of Computing and Data Science) and Rice University (Department of Psychological Sciences)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the evaluated models (GPT-4o-mini, Qwen2.5-7B, Qwen3-4B, Mistral-8B). This matters because SOTOPIA scenarios may be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SOTOPIA scenarios (published 2023) may have been seen during training of the evaluated models. Since SOTOPIA is public and all models were trained after 2023, this is a relevant concern."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SOTOPIA scenarios were published in 2023 and are publicly available. All evaluated models were likely trained after this date. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation study involving 6 annotators rating 120 scenarios is not pre-registered. No link to a pre-registration is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The Ethics Statement discusses ethical guidelines but does not mention IRB or ethics board approval for the human evaluation study."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only gender distribution is reported (50% women, 50% men) and institutional affiliation (two universities). No information on age, experience level, cultural background, or language proficiency — all relevant for evaluating communication barriers."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for annotators are stated beyond being from two universities."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human evaluation is an annotation task, not an experimental study with treatment conditions requiring randomization."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "It is not stated whether annotators knew the barrier conditions when rating metrics. The annotation UI (Figure 7) shows barrier definitions alongside the dialogue, which may bias ratings."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on whether all 6 annotators completed all assigned scenarios or whether any dropped out."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported despite running 720 scenarios across multiple models plus GPT-4o evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training used '4 × A6000 80G GPUs, across 20 epochs' (Appendix D.3). However, total GPU hours and API costs for the simulation and evaluation phases are not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds for the simulation experiments. Temperature 0.7 introduces stochasticity but seed sensitivity is not analyzed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "It is not stated whether results in Table 2 are from single runs or averaged across multiple runs. Standard deviations are shown but their source (across scenarios vs across runs) is ambiguous."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for the training experiments (BC+SR). The learning rate, batch size, and QLoRA parameters are stated but no search process is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for how the reported configuration was selected. Training hyperparameters appear chosen without documenting the selection process."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple significance tests are performed across models, barrier types, and metrics (Figures 4-5) without mention of correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own SocialVeil framework and barrier implementations without discussing potential bias in their own evaluation design."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "Compute differences between conditions are negligible — all use the same models and same number of dialogue turns."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 validates barrier construct validity through t-SNE visualization of model representations, showing barriers form distinct clusters. Section 5.1-5.2 validates through linguistic signatures and human evaluation alignment."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation framework (SocialVeil) IS the thing being tested. The same scaffolding is used across all model comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SOTOPIA scenarios (2023) predate all evaluated models' training. No temporal leakage analysis is performed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setup leaks information. The barrier agent's style prompts could provide hints about expected partner behavior."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All 180 episodes per condition are adapted from SOTOPIA scenarios. No discussion of potential dependencies between scenarios or overlap with training data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Communication barriers consistently impair LLM agents' social intelligence, with mutual understanding reduced by over 45% and confusion elevated by nearly 50%.",
       "evidence": "Table 2 shows consistent performance degradation across all four models and six metrics under all three barrier types compared to barrier-free baseline.",
       "supported": "strong"
     },
     {
       "claim": "Each barrier type produces a characteristic degradation pattern: semantic vagueness impairs mutual understanding most (avg -58%), emotional interference damages relationships most (avg -49%), and sociocultural mismatch induces persistent confusion (avg -49%).",
       "evidence": "Table 2 per-barrier results and Figure 5 showing per-barrier-type percentage deviations with statistical significance markers.",
       "supported": "strong"
     },
     {
       "claim": "Human evaluations validate the fidelity of simulated barriers with ICC≈0.78, Pearson r≈0.80, and barrier identification accuracy of 68%.",
       "evidence": "Section 5.2: ICC(1,k) = 0.77 for confusion, 0.79 for mutual understanding. Pearson r = 0.80 for confusion, 0.79 for mutual understanding. Bootstrap accuracy 0.68 [0.63, 0.73].",
       "supported": "strong"
     },
     {
       "claim": "Repair instruction yields trivial performance improvements while interactive learning (BC+SR) offers consistent but modest gains, with neither restoring baseline performance.",
       "evidence": "Table 3 shows repair instruction gains are negligible (e.g., GOAL 5.99→6.07 for semantic barrier) and BC+SR gains are modest (avg 10-20% improvement).",
       "supported": "strong"
     },
     {
       "claim": "Social reasoning is more fragile than goal pursuit, with relationship quality dropping by 45% and mutual understanding by 52% versus 20-30% for goal completion.",
       "evidence": "Section 4.2 finding 3, derived from Table 2 comparing degradation magnitudes across metric categories.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "SOTOPIA scenarios are publicly available since 2023 and all evaluated models were trained after this date. No analysis of whether models have seen these scenarios during training, which could inflate baseline performance."
     },
     {
       "flag": "GPT-4o as both barrier agent backbone and evaluator",
       "detail": "GPT-4o-mini serves as the barrier agent and GPT-4o serves as the evaluator. Using models from the same family for both generation and evaluation creates potential circularity in the evaluation pipeline."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "The paper performs many statistical tests across 4 models × 3 barrier types × 6 metrics and reports significance without any correction for multiple comparisons."
     },
     {
       "flag": "Limited annotator demographics",
       "detail": "Only 6 annotators from two universities, with minimal demographic reporting. For a study about sociocultural communication barriers, annotator cultural background and language proficiency are highly relevant but unreported."
     }
   ],
   "cited_papers": [
     {
       "title": "SOTOPIA: Interactive Evaluation for Social Intelligence in Language Agents",
       "authors": ["Xuhui Zhou", "Hao Zhu", "Leena Mathur"],
       "year": 2023,
       "arxiv_id": "2310.11667",
       "relevance": "Core benchmark from which SocialVeil adapts scenarios; key prior work on interactive evaluation of LLM social intelligence."
     },
     {
       "title": "SOTOPIA-π: Interactive Learning of Socially Intelligent Language Agents",
       "authors": ["Ruiyi Wang", "Haofei Yu", "Wenxin Zhang"],
       "year": 2024,
       "arxiv_id": "2403.08715",
       "relevance": "Interactive learning framework adapted for SocialVeil's BC+SR adaptation strategy."
     },
     {
       "title": "Generative Agents: Interactive Simulacra of Human Behavior",
       "authors": ["Joon Sung Park", "Joseph O'Brien"],
       "year": 2023,
       "relevance": "Foundational work on LLM-based social simulation and agent behavior."
     },
     {
       "title": "SocialBench: Sociality Evaluation of Role-Playing Conversational Agents",
       "authors": ["Hongzhan Chen"],
       "year": 2024,
       "arxiv_id": "2403.13679",
       "relevance": "Related benchmark for evaluating social intelligence in conversational AI agents."
     },
     {
       "title": "AgentSense: Benchmarking Social Intelligence of Language Agents through Interactive Scenarios",
       "authors": ["Xinyi Mou", "Jingcong Liang"],
       "year": 2024,
       "arxiv_id": "2410.19346",
       "relevance": "Independent interaction benchmark used for cross-benchmark transfer validation of SocialVeil."
     },
     {
       "title": "Why do Multi-Agent LLM Systems Fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Empirical analysis of multi-agent system failures, including communicative misalignment relevant to the survey scope."
     },
     {
       "title": "Communicative Agents for Software Development",
       "authors": ["Chen Qian", "Xin Cong"],
       "year": 2023,
       "arxiv_id": "2307.07924",
       "relevance": "Multi-agent collaborative software development using LLMs — relevant to agentic workflow evaluation."
     },
     {
       "title": "Using Large Language Models to Simulate Multiple Humans and Replicate Human Subject Studies",
       "authors": ["Gati V Aher", "Rosa I Arriaga", "Adam Tauman Kalai"],
       "year": 2023,
       "relevance": "Methodological work on using LLMs to simulate human subjects, relevant to agent evaluation methodology."
     },
     {
       "title": "QLoRA: Efficient Finetuning of Quantized LLMs",
       "authors": ["Tim Dettmers", "Artidoro Pagnoni"],
       "year": 2023,
       "relevance": "Training method used for the interactive learning adaptation strategy in SocialVeil."
     }
   ]
 }