ai-research-survey

scan.json (19196B)

---

 {
   "paper": {
     "title": "Compiler.next: A Search-Based Compiler to Power the AI-Native Future of Software Engineering",
     "authors": ["Filipe R. Cogo", "Gustavo A. Oliva", "Ahmed E. Hassan"],
     "year": 2025,
     "venue": "Manuscript submitted to ACM",
     "arxiv_id": "2510.24799"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided. The paper describes a prototype but does not release it."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses HumanEval-Plus (public benchmark), but no compilation traces, gold labels, or experimental data are released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided for the proof-of-concept experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report single point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims of improvement (e.g., 46.4% accuracy improvement) are made by comparing two numbers with no statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report percentage improvements with baseline context (e.g., accuracy from 0.26 to 0.56, latency from 14.2 to 10.8s)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Five generations of ten candidates per task with 70/30 train/test split of HumanEval-Plus is described but not justified. No discussion of whether this is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs. No variance, standard deviation, or multiple-run results are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The 'Initial' synthesized prompt serves as a baseline compared against the 'Optimized' prompt in Table 1. Table 2 compares with and without caching."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No comparison against other prompt optimization systems (DSPy, TextGrad, EvoPrompt, etc.) despite discussing them extensively in related work."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study isolating contributions of individual Compiler.next components (e.g., self-reflection, crossover, mutation, caching)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: accuracy, average latency, and average number of tokens."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a compiler/optimization system evaluated via automated code execution metrics; human evaluation is not relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "70% of HumanEval-Plus tasks used as gold labels for optimization, 30% held out for evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate results are reported in Tables 1 and 2. No per-task or per-category breakdown is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis of the compilation process or generated code is discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 shows caching reduces accuracy from 1.00 to 0.70 (-30%) and increases latency, honestly reporting the trade-off cost."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims Compiler.next enables 'seamless evolution of AI-native software systems' and 'democratizing software development' — these are unsupported vision statements far beyond what the small proof-of-concept demonstrates."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Compiler.next 'improves accuracy while also reducing latency and execution cost' (causal language) based on a single-run comparison of initial vs. optimized prompts with no controls for confounds."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation uses only HumanEval-Plus (Python code generation) with two models, but the paper positions Compiler.next as a general-purpose compiler for all FMware. The title claims 'AI-Native Future of Software Engineering' broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the observed improvements are discussed. For example, simple prompt engineering without the full compiler infrastructure might achieve similar results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models listed as 'Qwen2.5-7B-Instruct' and 'GPT-4o-mini' without snapshot dates or API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The intent string is given ('Generate source code from documentation') but the actual system prompts, mutation prompts, crossover prompts, and self-reflection prompts used by the compiler are not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Some hyperparameters are reported: 5 generations, 10 candidates per task, 70/30 split, Euclidean similarity threshold of 0.85 for caching."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The search mechanism, NSGA-II optimizer, caching architecture, and overall compilation pipeline are described in detail in Section 4 with figures."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how HumanEval-Plus tasks were split into the 70/30 partition or any preprocessing applied."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. The paper ends with a conclusion and disclaimer."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 states: 'intent compilation represents a foundational capability... but it is not the entirety of software engineering in the SE 3.0 era' and lists complementary advances needed. Section 3.6 positions the work relative to MBSE and self-adaptive systems."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data, compilation traces, or experimental logs are made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation data source (HumanEval-Plus benchmark) is clearly identified with the 70/30 split and evaluation procedure described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data is from a standard benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The high-level optimization loop is described but specific details of data flow (e.g., how many tasks total, which tasks in train vs. test) are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section. Two of three authors are from Huawei Canada but no funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: two from Centre for Software Excellence - Huawei Canada, one from Queen's University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Two authors are Huawei employees. Huawei has commercial interest in AI-native development tools, making the funder non-independent of the outcome. No disclosure of this conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Authors from Huawei may have financial interests in commercializing this technology."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for either Qwen2.5-7B-Instruct or GPT-4o-mini."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval-Plus is a well-known public benchmark likely in both models' training data. No discussion of potential contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in 2021 and is widely known. Both models were trained after this date. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token counts and latency per run are reported in Tables 1 and 2. Total compilation runtime is reported in Table 2 (8m:15s vs 10m:27s)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Total API cost, GPU usage for Qwen2.5-7B-Instruct, and overall computational budget are not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Compiler.next improves accuracy from 0.26 to 0.56 for Qwen2.5-7B-Instruct and from 0.68 to 1.00 for GPT-4o-mini on HumanEval-Plus",
       "evidence": "Table 1 in Section 4.3 shows these results comparing initial synthesized prompts vs. best optimized prompts after 5 generations.",
       "supported": "weak"
     },
     {
       "claim": "Compiler.next reduces latency and token usage while improving accuracy",
       "evidence": "Table 1 shows latency reduced from 14.2s to 10.8s (Qwen) and 8.7s to 5.0s (GPT-4o-mini), and tokens reduced from 537 to 369 (Qwen) and 500 to 417 (GPT-4o-mini).",
       "supported": "weak"
     },
     {
       "claim": "Semantic caching provides 22.1% speed-up in compilation time",
       "evidence": "Table 2 shows total running time of 10m:27s without cache vs 8m:15s with cache. However, caching reduced accuracy from 1.00 to 0.70.",
       "supported": "weak"
     },
     {
       "claim": "Compiler.next enables the seamless evolution of AI-native software systems as part of the SE 3.0 era",
       "evidence": "This is a vision claim supported by architectural descriptions and the proof-of-concept, but no evidence of 'seamless evolution' or real-world deployment.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": ["theoretical", "benchmark-eval", "case-study"],
   "key_findings": "Compiler.next is proposed as a search-based compiler that uses NSGA-II to optimize FMware configurations (prompts, model parameters, cognitive architectures) from human intents. A proof-of-concept on HumanEval-Plus shows the system can improve code generation accuracy for two models while reducing latency and token usage, though at the cost of exploration diversity when caching is enabled. The paper is primarily a vision paper with 10 calls for action for the SE community, with only a small-scale validation.",
   "red_flags": [
     {
       "flag": "Company evaluating its own vision",
       "detail": "Two of three authors are Huawei employees proposing a vision that aligns with Huawei's commercial interests in AI development platforms. No conflict of interest disclosure."
     },
     {
       "flag": "Vision claims far exceed evidence",
       "detail": "The paper claims to enable 'democratizing software development' and 'seamless evolution of AI-native software systems' based on a small proof-of-concept with 5 generations of 10 candidates on a single benchmark."
     },
     {
       "flag": "No comparison with existing tools",
       "detail": "Despite extensively discussing DSPy, TextGrad, EvoPrompt, ProTeGi, and SAMMO in related work, no experimental comparison is made against any of them."
     },
     {
       "flag": "Single-run results with no uncertainty quantification",
       "detail": "Tables 1 and 2 appear to report single-run results with no error bars, confidence intervals, or multiple-run statistics, making it impossible to assess result stability."
     },
     {
       "flag": "Benchmark contamination risk",
       "detail": "HumanEval-Plus (published 2021) is used without any discussion of contamination for models trained well after its publication."
     },
     {
       "flag": "GPT-4o-mini reaches 100% accuracy",
       "detail": "GPT-4o-mini achieves 1.00 accuracy on the 30% held-out HumanEval-Plus tasks, which is suspiciously high and could indicate contamination or a very easy subset."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig", "S. Yao", "K. Pei", "O. Press", "K. Narasimhan"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM coding agents on real-world software engineering tasks."
     },
     {
       "title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines",
       "authors": ["O. Khattab", "A. Singhvi", "P. Maheshwari"],
       "year": 2023,
       "arxiv_id": "2310.03714",
       "relevance": "Key prior work on programmatic optimization of LLM pipelines, directly comparable to Compiler.next."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Q. Wu", "G. Bansal", "J. Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Multi-agent framework for LLM applications, relevant to agentic AI architectures."
     },
     {
       "title": "Automated Design of Agentic Systems",
       "authors": ["S. Hu", "C. Lu", "J. Clune"],
       "year": 2024,
       "arxiv_id": "2408.08435",
       "relevance": "Automated search over agent architectures, directly relevant to the survey's coverage of agentic system design."
     },
     {
       "title": "Towards AI-Native Software Engineering (SE 3.0): A Vision and a Challenge Roadmap",
       "authors": ["A. E. Hassan", "G. A. Oliva", "D. Lin"],
       "year": 2024,
       "arxiv_id": "2410.06107",
       "relevance": "Foundational vision paper for SE 3.0 paradigm that Compiler.next builds upon."
     },
     {
       "title": "Large Language Models Are Human-Level Prompt Engineers",
       "authors": ["Y. Zhou", "A. I. Muresanu", "Z. Han"],
       "year": 2023,
       "relevance": "Automatic Prompt Engineer (APE) — foundational work on automated prompt optimization using LLMs."
     },
     {
       "title": "Promptbreeder: Self-Referential Self-Improvement via Prompt Evolution",
       "authors": ["C. Fernando", "D. S. Banarse", "H. Michalewski"],
       "year": 2024,
       "relevance": "Evolutionary prompt optimization with self-referential mutation, directly relevant to LLM prompt engineering methodology."
     },
     {
       "title": "Connecting Large Language Models with Evolutionary Algorithms Yields Powerful Prompt Optimizers",
       "authors": ["Q. Guo", "R. Wang", "J. Guo"],
       "year": 2024,
       "relevance": "EvoPrompt — evolutionary approach to prompt optimization, comparable methodology to Compiler.next."
     },
     {
       "title": "Optimizing Generative AI by Backpropagating Language Model Feedback",
       "authors": ["M. Yuksekgonul", "F. Bianchi", "J. Boen"],
       "year": 2025,
       "doi": "10.1038/s41586-025-08661-4",
       "relevance": "TextGrad — gradient-descent metaphor for optimizing LLM pipelines, key competing approach."
     },
     {
       "title": "Rethinking Software Engineering in the Era of Foundation Models: A Curated Catalogue of Challenges in the Development of Trustworthy FMware",
       "authors": ["A. E. Hassan", "D. Lin", "G. K. Rajbahadur"],
       "year": 2024,
       "doi": "10.1145/3663529.3663849",
       "relevance": "Foundational work on challenges in FMware engineering that Compiler.next addresses."
     }
   ]
 }