ai-research-survey

scan.json (24665B)

---

 {
   "paper": {
     "title": "Prompt Less, Smile More: MTP with Semantic Engineering in Lieu of Prompt Engineering",
     "authors": [
       "Jayanaka L. Dantanarayana",
       "Savini Kashmira",
       "Thakee Nathees",
       "Zichen Zhang",
       "Krisztian Flautner",
       "Lingjia Tang",
       "Jason Mars"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.19427",
     "doi": "10.48550/arXiv.2511.19427"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Semantic Engineering via SemTexts improves MTP performance by 1.3x to 3x over base MTP on complex AI-integrated benchmarks, matching or surpassing manually crafted Prompt Engineering while requiring ~3.8x less developer effort (measured in LOC). An ablation study shows that a few targeted SemText annotations at semantic gap points are most effective, with diminishing returns from additional annotations. Spatial affinity between semantic context and code entities in prompts significantly improves LLM reasoning accuracy.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The implementation is described as extending the Jac language but no link to the source code is given."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Custom synthetic datasets (Memory Retrieval with 60 users/300 queries, Task Manager with 220 queries) are not released. While some evaluation uses public datasets (LAION-400M, InstructEval, SWE-bench Lite), the custom datasets have no download link."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (NVIDIA RTX 3090, 24GB VRAM, 64GB RAM) but no requirements.txt, Dockerfile, or detailed dependency list is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports point estimates only (e.g., '0.701', '89.546%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements (e.g., '1.3x to 3x') by comparing raw numbers in Table 2 without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvement factors (1.3x to 3x) with baseline context. Table 2 provides absolute scores for all three conditions, and Table 3 reports LOC reduction factors (e.g., '↓×3.53'). Readers can assess magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for dataset sizes (300 images, 220 queries, 200 prompts, 300 SWE-bench tasks). No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are included: traditional Prompt Engineering (PE) and base MTP without SemTexts (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MTP (2025, same group) is the most recent AI-integration framework. PE is the standard practice. DSPy is discussed as related work. Baselines are appropriate."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3.1 presents a detailed ablation study on Content Creator, incrementally adding SemTexts (MTPbase → Semrouting → Semstage → Semreview → Semall → Semprompt) and measuring each step's contribution (Figure 8)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Each benchmark uses only a single metric: F1 for Memory Retrieval, hybrid similarity for Image Extraction, success rate for Task Manager and Content Creator, test passing rate for Aider Genius. No benchmark is evaluated with multiple metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. Content Creator and Task Manager use LLM-as-a-judge. All evaluation is automated."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit separation of dev and test splits is described. It is unclear whether the same data was used for development/tuning and final evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-benchmark breakdowns for all three methods across two models. The ablation study (Figure 8) shows step-by-step performance changes. Content Creator evaluation includes four writing categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3.1 discusses routing errors in MTP's failing cases and Section 5.3.2 discusses the Memory Retrieval case where SemTexts provide no benefit. The Semprompt ablation step shows degraded performance."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: SemTexts offer no benefit on Memory Retrieval (Section 5.3.2), Semprompt reduces accuracy compared to Semreview (Section 5.3.1, Figure 8), and adding SemTexts to all entities (Semall) provides no gain beyond targeted placement."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '1.3x to 3x' improvement are supported by Table 2 results. '3.8x' effort reduction is supported by Table 3. Claims of matching PE are supported across benchmarks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('SemTexts improve MTP performance') are supported by controlled ablation studies (Section 5.3.1) where single-variable manipulation isolates the contribution of each SemText group."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents 'Semantic Engineering' as a general paradigm (title: 'in Lieu of Prompt Engineering') but evaluates only within the Jac/MTP framework with two models (GPT-4o, Gemma3:27b). No evidence it generalizes beyond MTP."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For example, improvements could stem from providing more text context rather than the specific SemText mechanism. The LLM-as-judge evaluation could also favor longer/structured outputs."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy/success rates and claims performance improvement — the measurements match the granularity of the claims. Developer effort is measured in LOC, which is acknowledged as a proxy following prior work [3, 7]."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper says 'GPT-4o' without a snapshot date or API version, and 'Gemma3:27b' without a specific version identifier. Marketing name 'GPT-4o' does not count as a specified version per schema guidance."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The PE prompt is shown in full (Figure 3, 70 lines). The MTP-generated prompt is shown (Figure 4c). SemText annotations are shown (Figures 7, 9). The prompt generation mechanism is formally specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM inference hyperparameters (temperature, top-p, max tokens) are reported for either GPT-4o or Gemma3:27b."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The MTP compilation pipeline and runtime are described in detail (Section 4, Algorithms 1-2, Figure 5). The SemTable build pass and enriched MT-IR construction are formally specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Synthetic dataset generation for Memory Retrieval and Task Manager is described only at a high level (Table 1). No details on how the synthetic data was created, filtered, or validated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what settings or scenarios the results do NOT apply to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual benchmark results, LLM outputs) is available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Custom synthetic datasets are described only briefly in Table 1. No details on generation methodology for the Memory Retrieval (60 users, 300 queries) or Task Manager (220 queries) datasets."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (SWE-bench Lite, InstructEval, LAION-400M) and synthetic datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of data pipeline from collection to final analysis. Unclear how synthetic datasets were validated or how benchmark subsets were selected (e.g., 300 from LAION-400M, 300 from SWE-bench Lite)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. An Acknowledgments section mentions no grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Michigan and Jaseci Labs. Jaseci Labs develops the Jac programming language being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. One author (Thakee Nathees) is from Jaseci Labs, which develops Jac, the language being evaluated. This conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Jaseci Labs (author affiliation) commercially develops the Jac language being evaluated, but no financial interest disclosure is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-4o or Gemma3:27b."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench Lite (from GitHub issues) could overlap with GPT-4o's training data. This is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench Lite was published in 2024 and uses pre-existing GitHub issues. InstructEval was published in 2023. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency numbers are reported despite using GPT-4o API and running Gemma locally."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (RTX 3090, 64GB RAM) but no total compute budget, GPU hours, or API spend is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed experiments. Results appear to be single-run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results in Table 2 have no indication of how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The SemText content appears manually crafted but no systematic exploration is documented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the final SemText configurations were selected. The ablation shows incremental additions but no systematic selection criterion."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own system (Jac/MTP/SemTexts) against their own PE baselines. No acknowledgment of author-evaluation bias. All baselines were implemented by the same team."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "SemTexts add compilation and prompt generation overhead but performance vs. compute tradeoff is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Benchmark Applications) explicitly discusses why GSM8K and HotpotQA are insufficient benchmarks for AI-integrated applications and introduces new benchmarks designed to test capabilities C1-C6 that better represent real-world scenarios."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The scaffold (MTP vs PE) IS the thing being compared — the paper is evaluating different prompt generation approaches, not comparing models across scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o or Gemma may have seen SWE-bench, InstructEval, or LAION-400M data during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between training data and benchmark examples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "SemTexts improve MTP performance by 1.3x to 3x on complex benchmarks.",
       "evidence": "Table 2 shows improvements across benchmarks: Task Manager 36.8% → 92.3% (2.5x), Content Creator 32.5% → 96.0% (2.95x), Image Extraction 0.284 → 0.439 (1.55x), Aider Genius 9.7% → 18.7% (1.93x) with GPT-4o.",
       "supported": "strong"
     },
     {
       "claim": "MTP+SemTexts matches or surpasses Prompt Engineering performance across benchmarks.",
       "evidence": "Table 2 shows MTP+SemTexts achieves comparable scores to PE: Task Manager 92.3% vs 89.5%, Content Creator 96.0% vs 95.0%, Image Extraction 0.439 vs 0.427, Aider Genius 18.7% vs 19.7%.",
       "supported": "moderate"
     },
     {
       "claim": "Semantic Engineering reduces developer effort by ~3.8x compared to Prompt Engineering.",
       "evidence": "Table 3 reports LOC reduction factors averaging 3.8x across 5 benchmarks (ranging from 1.76x to 7.8x).",
       "supported": "moderate"
     },
     {
       "claim": "SemTexts outperform docstrings as semantic annotation mechanisms by 7-8% on benchmark accuracy.",
       "evidence": "Table 4 shows MTP+SemText vs MTP+Docstring: Content Creator 96% vs 88%, Task Manager 92.3% vs 85.7%. Attributed to spatial affinity of semantic context (Section 5.4).",
       "supported": "moderate"
     },
     {
       "claim": "Accuracy does not increase linearly with the number of SemTexts; targeted placement at semantic gap points is most effective.",
       "evidence": "Ablation study (Figure 8) shows Semall and Semprompt provide no improvement or even degrade performance compared to targeted SemTexts (Semreview).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Authors evaluate own system",
       "detail": "All authors are from the team that developed Jac/MTP. They evaluate their own system (MTP+SemTexts) against their own PE baselines and their own custom benchmarks, without acknowledging author-evaluation bias. All benchmarks were designed by the same group."
     },
     {
       "flag": "Custom benchmarks without validation",
       "detail": "Three of five benchmarks (Memory Retrieval, Task Manager, Content Creator) use custom synthetic datasets created by the authors. No external validation of these benchmarks. The benchmarks may be designed to favor the SemText approach."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "No error bars, confidence intervals, variance, or multi-run results are reported for any benchmark. LLM outputs are stochastic, so single-run results may not be reproducible."
     },
     {
       "flag": "LLM-as-judge without validation",
       "detail": "Task Manager and Content Creator use LLM-as-judge evaluation without validating judge accuracy against human judgments. The judge may systematically favor certain output styles."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations section, no threats to validity, and no scope boundaries — unusual for a systems paper making broad claims about a 'new programming paradigm'."
     },
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "One author is from Jaseci Labs, which commercially develops the Jac language being evaluated. This conflict is not acknowledged, and no competing interests statement is included."
     }
   ],
   "cited_papers": [
     {
       "title": "Prompting Is Programming: A Query Language for Large Language Models",
       "authors": ["Luca Beurer-Kellner", "Marc Fischer", "Martin Vechev"],
       "year": 2023,
       "doi": "10.1145/3591300",
       "relevance": "LMQL framework for constrained LLM generation — direct competitor in AI-integrated programming paradigm."
     },
     {
       "title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines",
       "authors": ["Omar Khattab", "Arnav Singhvi", "Paridhi Maheshwari"],
       "year": 2023,
       "arxiv_id": "2310.03714",
       "relevance": "Major framework for automated prompt generation that reduces prompt engineering overhead — primary comparison point."
     },
     {
       "title": "MTP: A Meaning-Typed Language Abstraction for AI-Integrated Programming",
       "authors": ["Jayanaka L. Dantanarayana", "Yiping Kang"],
       "year": 2025,
       "doi": "10.1145/3763092",
       "relevance": "The foundational MTP system that this paper extends — core AI-integrated programming abstraction."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "Benchmark used for Aider Genius evaluation — tests real-world code editing capability."
     },
     {
       "title": "SGLang: Efficient Execution of Structured Language Model Programs",
       "authors": ["Lianmin Zheng", "Liangsheng Yin"],
       "year": 2024,
       "arxiv_id": "2312.07104",
       "relevance": "Efficient structured LLM program execution framework — related approach to AI-integrated programming."
     },
     {
       "title": "Effective LLM-Driven Code Generation with Pythoness",
       "authors": ["Kyla H. Levin", "Kyle Gwilt", "Emery D. Berger", "Stephen N. Freund"],
       "year": 2025,
       "arxiv_id": "2501.02138",
       "relevance": "LLM-driven runtime code generation approach — alternative paradigm for AI-integrated programming."
     },
     {
       "title": "A Systematic Survey of Prompt Engineering in Large Language Models: Techniques and Applications",
       "authors": ["Pranab Sahoo", "Ayush Kumar Singh"],
       "year": 2025,
       "arxiv_id": "2402.07927",
       "relevance": "Comprehensive survey of prompt engineering techniques — contextualizes the problem SemTexts addresses."
     },
     {
       "title": "PromptPex: Automatic Test Generation for Language Model Prompts",
       "authors": ["Reshabh K Sharma", "Jonathan De Halleux", "Shraddha Barke", "Benjamin Zorn"],
       "year": 2025,
       "arxiv_id": "2503.05070",
       "relevance": "Automatic testing of LLM prompts — relevant to quality assurance of AI-integrated systems."
     },
     {
       "title": "GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning",
       "authors": ["Lakshya A Agrawal", "Shangyin Tan"],
       "year": 2025,
       "arxiv_id": "2507.19457",
       "relevance": "Evolutionary prompt optimization built on DSPy — alternative approach to improving automated prompts."
     }
   ]
 }