ai-research-survey

scan.json (21977B)

---

 {
   "paper": {
     "title": "Automating REST API Postman Test Cases Using LLM",
     "authors": [
       "S Deepika Sri",
       "Mohammed Aadil S",
       "Sanjjushri Varshini R",
       "Raja CSP Raman",
       "Gopinath Rajagopal",
       "S Taranath chan"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2404.10678",
     "doi": "10.48550/arXiv.2404.10678"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub link is provided at the top of the paper: https://github.com/tactlabs/test-case-generation."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states the model is 'trained using manually collected postman test cases' but no dataset is released or linked. The GitHub repo is mentioned but no data download link is provided in the paper."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is mentioned in the paper. The tech stack section mentions Flask, Postman, and OpenAI but provides no library versions or dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the architecture at a high level (training phase, testing phase) but provides no commands, configuration steps, or concrete instructions for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative results are reported at all. The Results section (Section 5) is entirely qualitative, describing screenshots of the interface rather than any measured outcomes. No confidence intervals or error bars appear anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The paper makes claims about efficacy and efficiency but provides no comparative numerical results and no statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes or quantitative improvements are reported. The paper contains no numerical performance metrics of any kind."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'manually collected postman test cases' for training but never states how many test cases were collected or why that number is sufficient. No sample size information is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or any measure of variability is reported. There are no quantitative results at all."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included. The paper does not compare its approach against any prior method, manual test case writing, or alternative tool. The Results section shows only screenshots of the system's interface."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so the question of whether they are contemporary is moot. The literature review mentions RESTTESTGEN and other tools but none are used for comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The system has multiple components (data collection, preprocessing, OpenAI model, Flask interface) but none are varied or removed to assess their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No evaluation metrics of any kind are reported. The Results section contains no quantitative measurements — only qualitative descriptions and screenshots."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated test cases is reported. There is no user study, expert review, or manual assessment of the quality or correctness of the generated outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No held-out test set is mentioned. The paper does not describe any train/test split or evaluation on unseen data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No per-category or per-API breakdown is provided. Results are not reported in any granular way — there are no tables or figures with quantitative data."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases are discussed. The paper presents only a positive narrative about the system's capabilities without any discussion of where the approach fails or produces incorrect test cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every description is positive, with no mention of approaches that failed, limitations encountered during development, or configurations that did not work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the approach 'enhance[s] the efficiency and effectiveness of test case generation' and provides 'comprehensive testing,' but the Results section contains no quantitative evidence supporting these claims — only screenshots of the web interface."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'LLMs enhance the creation of Postman test cases' and the methodology 'streamlines the test case generation process.' No controlled experiment or adequate study design supports these causal assertions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims about 'comprehensive testing' and 'a wide range of REST API properties' without bounding these generalizations to the specific APIs tested. No scope limitations on the types of REST APIs or domains are stated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The paper does not consider whether simpler approaches (e.g., template-based test generation) could achieve similar results, or whether observed outputs are due to OpenAI's general capabilities rather than the proposed methodology."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'OpenAI's language models, such as GPT-3' in Section 4.3 but does not specify an exact model version, API version, or snapshot date. The reference to GPT-3 is illustrative rather than specifying which model was actually used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No prompts or system instructions are provided. The paper describes the approach at a high level but never shows the actual prompt text sent to the OpenAI model for generating test cases."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported. Temperature, top-p, max tokens, learning rate, or any other configuration parameters are not mentioned."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a system with training, loading, and inference phases but provides insufficient detail about the actual scaffolding — no specifics on how the OpenAI API is called, how prompts are constructed from user input, or how outputs are post-processed into Postman format."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1.3 mentions 'tokenization, normalization, and potentially data augmentation' in generic terms but provides no concrete details about what preprocessing was actually performed on the collected Postman test cases."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations or threats-to-validity section. The paper has a Future Scope section (Section 6) that discusses extensions but does not discuss any limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no acknowledgment of potential weaknesses or methodological concerns."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No scope boundaries are stated. The paper does not specify what types of REST APIs were tested, what the approach cannot handle, or what claims the authors are not making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw training data (manually collected Postman test cases) is not made available. No data files, download links, or supplementary materials are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper says data was 'manually collected postman test cases or instances for various Rest APIs' but provides no details on how many were collected, from which APIs, by whom, or over what time period."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data consists of Postman test cases, not human subject data. The paper is a benchmark/tool evaluation, not a human study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The data pipeline is described only in generic terms (Section 3.1: collection, storage, preprocessing, training, storage). No concrete details are given about the number of examples at each stage, filtering criteria, or actual transformations applied."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "Author names and email addresses are listed but no institutional affiliations are provided. It is unclear whether the authors are affiliated with any company or university."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The absence of any funding or conflict-of-interest statement means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses OpenAI models (GPT-3 mentioned) to generate test cases but does not state the model's training data cutoff date. This matters because the model may have seen Postman test case patterns during pre-training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the OpenAI model may have seen similar Postman test cases during its pre-training. The paper fine-tunes or prompts an LLM but does not address potential overlap between training data and evaluation scenarios."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No benchmark contamination is discussed. The paper does not consider whether the REST APIs used for evaluation or their test patterns could have been in the LLM's pre-training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It is a tool/system paper with no user study component."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, tokens consumed, or latency figures are reported. The system calls OpenAI's API but the paper does not quantify the cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. Training time, hardware used, or total API spend are not mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "The LLM-based approach enhances the efficiency and effectiveness of test case generation for REST APIs.",
       "evidence": "No quantitative evidence is provided. The Results section (Section 5) shows screenshots of the web interface but reports no metrics comparing efficiency or effectiveness to any baseline.",
       "supported": "unsupported"
     },
     {
       "claim": "LLMs can intelligently formulate test cases that cover a broad range of REST API properties, ensuring comprehensive testing.",
       "evidence": "No measurement of coverage breadth is provided. The paper shows example output screenshots but does not quantify what proportion of API properties are covered or define 'comprehensive.'",
       "supported": "unsupported"
     },
     {
       "claim": "The approach reduces the workload for developers and testers.",
       "evidence": "No user study, time comparison, or workload measurement is provided. This claim is asserted without evidence in Section 5.",
       "supported": "unsupported"
     },
     {
       "claim": "The model conforms to current technological standards and holds promise for future advancements.",
       "evidence": "No evidence is provided for conformance to standards. This is a subjective assertion in the abstract and conclusion without supporting data.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": [
     "case-study"
   ],
   "key_findings": "The paper proposes a system that uses OpenAI's LLMs to automatically generate Postman test cases for REST APIs. The system is implemented as a Flask web application where users input a REST API endpoint and the desired number of test cases. The paper provides no quantitative evaluation of the generated test cases — the Results section consists entirely of screenshots of the web interface. No metrics on test case quality, coverage, correctness, or comparison to baselines are reported.",
   "red_flags": [
     {
       "flag": "No quantitative evaluation",
       "detail": "The Results section (Section 5) contains only screenshots and qualitative descriptions. There are zero quantitative metrics, tables, or figures with data. The paper makes claims about 'efficiency,' 'effectiveness,' and 'comprehensive testing' without any measurements."
     },
     {
       "flag": "No baselines or comparisons",
       "detail": "The paper does not compare its approach to any baseline — not manual test writing, not existing tools like RESTTESTGEN (which is cited in the literature review), not even a random generation approach."
     },
     {
       "flag": "Claims far exceed evidence",
       "detail": "The abstract and conclusion make broad claims about enhancing efficiency, comprehensive coverage, and conforming to technological standards, but the paper provides no evidence for any of these assertions."
     },
     {
       "flag": "Vague methodology",
       "detail": "The Proposed Solution (Section 3) describes generic machine learning training/testing phases using textbook descriptions (backpropagation, gradient descent, loss functions) but provides no specifics about the actual implementation — what model was used, how it was fine-tuned, what the prompts look like, or what the training data contains."
     },
     {
       "flag": "Missing model specification",
       "detail": "Section 4.3 mentions 'GPT-3' as an example of OpenAI models but never clarifies which model was actually used. The description of 'fine-tuning the pre-trained model' vs. the mention of APIs suggests confusion about whether the approach involves fine-tuning or prompting."
     },
     {
       "flag": "No limitations discussed",
       "detail": "The paper has no limitations section and does not acknowledge any weaknesses. The Future Scope section discusses extensions but not current shortcomings."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language model assisted software engineering: prospects, challenges, and a case study",
       "authors": ["Lenz Belzner", "Thomas Gabor", "Martin Wirsing"],
       "year": 2023,
       "relevance": "Surveys LLM applications across the software engineering lifecycle, relevant to the scope of AI-assisted programming."
     },
     {
       "title": "Generate and pray: Using sallms to evaluate the security of llm generated code",
       "authors": ["Mohammed Latif Siddiq", "Joanna Santos"],
       "year": 2023,
       "arxiv_id": "2311.00889",
       "relevance": "Proposes a framework to systematically assess LLMs' ability to generate secure code, relevant to LLM code generation quality evaluation."
     },
     {
       "title": "TestSpark: IntelliJ IDEA's Ultimate Test Generation Companion",
       "authors": ["Arkadii Sapozhnikov"],
       "year": 2024,
       "arxiv_id": "2401.06580",
       "relevance": "An IDE plugin supporting LLM-based test generation techniques, directly relevant to automated testing with LLMs."
     },
     {
       "title": "Exploring the effectiveness of large language models in generating unit tests",
       "authors": ["Mohammed Latif Siddiq"],
       "year": 2023,
       "arxiv_id": "2305.00418",
       "relevance": "Empirical study of LLM effectiveness in generating JUnit tests, relevant to evaluating LLM code generation capabilities."
     },
     {
       "title": "Effective test generation using pre-trained large language models and mutation testing",
       "authors": ["Arghavan Moradi Dakhel"],
       "year": 2024,
       "relevance": "Evaluates LLM-generated test quality using mutation testing, relevant to methodology for assessing LLM-generated code."
     },
     {
       "title": "Automated unit test improvement using large language models at Meta",
       "authors": ["Nadia Alshahwan"],
       "year": 2024,
       "arxiv_id": "2402.09171",
       "relevance": "Industry-scale evaluation of LLMs for unit test improvement at Meta, relevant to practical deployment of AI coding tools."
     },
     {
       "title": "Resttestgen: automated black-box testing of restful apis",
       "authors": ["Emanuele Viglianisi", "Michael Dallago", "Mariano Ceccato"],
       "year": 2020,
       "relevance": "Prior work on automated REST API test generation using Swagger definitions, a direct baseline for this paper's domain."
     },
     {
       "title": "Empowering llm to use smartphone for intelligent task automation",
       "authors": ["Hao Wen"],
       "year": 2023,
       "arxiv_id": "2308.15272",
       "relevance": "AutoDroid system integrating LLMs for mobile task automation, relevant to agentic AI applications."
     },
     {
       "title": "Jailbreaker: Automated jailbreak across multiple large language model chatbots",
       "authors": ["Gelei Deng"],
       "year": 2023,
       "arxiv_id": "2307.08715",
       "relevance": "Automated jailbreak attacks on LLM chatbots, relevant to AI safety and security evaluation."
     }
   ]
 }