ai-research-survey

scan.json (16916B)

---

 {
   "paper": {
     "title": "Automatically Surfacing Opportunities for Improvements In Internet-Scale Applications",
     "authors": ["Vipul Harsh", "Sayan Sinha", "Henry Milner", "Haijie Wu", "B Aditya Prakash", "Vyas Sekar", "Hui Zhang"],
     "year": 2025,
     "venue": "HotNets '25 (24th ACM Workshop on Hot Topics in Networks)",
     "doi": "10.1145/3772356.3772423"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses anonymized production data from a monitoring service provider but does not release it."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are included. The proof-of-concept implementation is described at a high level only."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses heuristic thresholds (2x rate, 25% absolute difference) for opportunity detection but no formal significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes reported. Results are presented as counts of useful opportunities (e.g., '5 out of 18 scenarios') without quantified magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "One week of production data from 3 services is used with no justification for why this duration or number of services is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation reported across any results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 compares categories of related work qualitatively but no quantitative baseline comparison is performed against any existing system."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative baselines are included at all."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (hypothesis generator, attribute computation, opportunity finder with two experts) but no ablation is performed."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is limited to counting how many opportunities were manually deemed useful. No other metrics (precision, recall, latency per hypothesis, etc.) are formally reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Results were manually inspected: 'we (manually) deemed to be insightful' (Section 5.2). However, the evaluation protocol is informal."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a benchmark evaluation paper; the system discovers opportunities in production data, so a held-out test set is not structurally applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-KPI breakdown of identified opportunities. Results are also broken down by Expert 1 (18 scenarios) and Expert 2 (20 scenarios)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges false positives: 'the false positive rate may seem high at first' and notes 13 of 18 Expert 1 scenarios and ~12 of 20 Expert 2 scenarios were not useful."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that many surfaced opportunities were false positives (only 5/18 from Expert 1 and 8/20 from Expert 2 were useful)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'early promise from a proof-of-concept system' with 'evaluation on three real-world services,' which is appropriately hedged and matches the results in Section 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-sounding claims (e.g., 'CORS error triggered client-side retries that led to the timeout') but acknowledges in Section 6 that causal inference methods are future work. The opportunity finder uses correlation-based heuristics only."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful to frame results as preliminary: 'proof of concept,' 'early promise,' and explicitly lists open challenges in Section 6. Claims are bounded to the three tested services."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive discussion of alternative explanations for the observed opportunities or why the heuristic thresholds might produce misleading results beyond acknowledging false positives."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'Llama 3.2' for hypothesis generation (Section 5.1), citing the model paper [18]. However, no specific parameter count or snapshot is given, so this is borderline. The citation to the Llama 3 paper provides traceability."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompt intent ('find patterns associated with high rebuffering') but does not provide the actual prompt text used with the LLM."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, etc.) are reported. Heuristic thresholds (10%, 2x, 25%) are stated but LLM settings are not."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-component pipeline (hypothesis generator, attribute computation engine, opportunity finder) is described with data flow in Figure 2 and Sections 4.1-4.3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'anonymized data' and 'one week of production data' but does not describe how the data was preprocessed, filtered, or anonymized."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Discussion and future work' identifies multiple open challenges and limitations of the current approach."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 discusses future work directions but does not identify specific threats to validity of the current evaluation (e.g., manual evaluation bias, threshold sensitivity, generalizability to other domains)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states it presents a 'vision' and 'proof of concept' and identifies specific things not yet addressed: causal inference, complex event patterns, scalable data processing, privacy."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw production data is not available; it is proprietary and anonymized."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the data source: 'anonymized data from a large application-level monitoring and analytics service provider' collecting client-side events from end-user devices."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data comes from production telemetry systems, not recruited subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw events to opportunities is described architecturally but specifics of data transformations, filtering steps, and intermediate counts are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Conviva (industry), Georgia Tech, and Carnegie Mellon University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Multiple authors are affiliated with Conviva, a commercial monitoring/analytics company. The system is evaluated on production data from what appears to be Conviva's service. This conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. Conviva-affiliated authors evaluating a system in their own production environment is a potential conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. The LLM is used as a component for hypothesis generation, not evaluated for its knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of model capabilities."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not a benchmark evaluation of model capabilities."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 reports processing times: Expert 1 processed >1000 hypotheses in <10 minutes; Expert 2 processed 250 hypotheses in <3 minutes per KPI."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget, hardware specifications, or LLM API costs are reported."
       }
     }
   },
   "claims": [
     {
       "claim": "The proof-of-concept system can surface novel opportunities for improvement in real-world production services.",
       "evidence": "Table 2 lists 8 specific opportunity leads across 3 production services, identified from one week of data (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "Expert 1 surfaced useful opportunities in 5 out of 18 scenarios; Expert 2 in 8 out of 20 scenarios on average.",
       "evidence": "Section 5.2 reports these counts, with manual inspection determining usefulness.",
       "supported": "weak"
     },
     {
       "claim": "The system is scalable: Expert 1 processed >1000 hypotheses in <10 minutes, Expert 2 processed 250 in <3 minutes per KPI.",
       "evidence": "Section 5.2 'Scalability' paragraph provides these timing numbers.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["case-study"],
   "key_findings": "The paper presents a vision and proof-of-concept for automatically surfacing improvement opportunities in Internet-scale services by generating and testing hypotheses based on derived attributes (indirect, stateful, non-local). Evaluated on production data from 3 services, the prototype identified actionable opportunities such as event sequences leading to payment failures and differential error rates for returning users. The system uses LLM-assisted hypothesis generation (Llama 3.2), efficient stateful attribute computation, and a mixture-of-experts validation approach, though the current evaluation is preliminary with high false positive rates (5/18 and 8/20 useful).",
   "red_flags": [
     {
       "flag": "Undisclosed conflict of interest",
       "detail": "Multiple authors are Conviva employees evaluating a system on what appears to be Conviva's production data. This commercial interest in the system's success is not acknowledged."
     },
     {
       "flag": "No quantitative baselines",
       "detail": "Despite Table 1 listing four categories of related work, no quantitative comparison against any existing system is performed."
     },
     {
       "flag": "Informal evaluation methodology",
       "detail": "The evaluation relies on manual inspection by the authors themselves to determine which opportunities are 'insightful.' No inter-rater agreement, blinding, or systematic evaluation criteria are described."
     },
     {
       "flag": "Selective reporting of opportunities",
       "detail": "Table 2 reports 'a subset of the opportunities' that were manually deemed insightful, without systematic criteria for selection."
     }
   ],
   "cited_papers": [
     {
       "title": "InsightPilot: An LLM-Empowered Automated Data Exploration System",
       "authors": ["P. Ma", "R. Ding", "S. Wang", "S. Han", "D. Zhang"],
       "year": 2023,
       "relevance": "LLM-based automated data analysis system, relevant to AI-assisted analytics and software tooling."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["M. Cemri", "M. Z. Pan", "S. Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Directly relevant to understanding failure modes in multi-agent LLM systems."
     },
     {
       "title": "Automatic root cause analysis via large language models for cloud incidents",
       "authors": ["Y. Chen", "H. Xie", "M. Ma"],
       "year": 2024,
       "relevance": "LLM-based root cause analysis for cloud systems, relevant to AI-assisted software engineering and operations."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["A. Grattafiori", "A. Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Foundation model used in this system's hypothesis generation component."
     },
     {
       "title": "A survey on multimodal large language models",
       "authors": ["S. Yin", "C. Fu", "S. Zhao"],
       "year": 2024,
       "relevance": "Survey of multimodal LLMs referenced as future direction for hypothesis generation."
     }
   ]
 }