ai-research-survey

scan.json (20776B)

---

 {
   "paper": {
     "title": "CoPrompter: User-Centric Evaluation of LLM Instruction Alignment for Improved Prompt Engineering",
     "authors": ["Ishika Joshi", "Simra Shahid", "Shreeya Venneti", "Manushree Vasu", "Yantao Zheng", "Yunyao Li", "Balaji Krishnamurthy", "Gromit Yeuk-Yin Chan"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2411.06099"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The system is described but no link to source code is given."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset, survey responses, or study data are released. The formative study and user evaluation data are not made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions React frontend and FastAPI backend, and GPT-4o for evaluation, but no requirements.txt, Dockerfile, or detailed dependency specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The system design is described at a high level but there are no instructions for replicating the tool or experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "SUS scores are reported as a mean (81.25) and range but no confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims about CoPrompter improving alignment are based on qualitative observations and SUS scores without any formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Improvements are described qualitatively (e.g., participants 'saw improved alignment scores') without quantified magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The user evaluation has N=8. The paper states the small sample is due to the requirement for experienced prompt engineers and 75-90 minute sessions, but no formal justification or power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "SUS scores are reported as a mean and range (Figure 9) but no standard deviation or variance is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No formal baseline comparison is included. The comparison to 'traditional methods' is based on participant interviews about their past experience, not a controlled comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "Related tools like ChainForge, SPADE, EvalLM, and EvalGen are discussed in related work but not used as experimental baselines."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "CoPrompter has multiple components (criteria generation, update module, alignment report card) but no ablation study examines which components contribute most."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses SUS scores (usability and efficiency subscales) and qualitative thematic analysis of user sessions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The core evaluation is a user study with 8 industry prompt engineers who used CoPrompter and provided qualitative feedback and SUS ratings (Section 6-7)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a user study of a tool, not a benchmark evaluation. No held-out test set is applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "SUS scores are broken down into System Usability and System Efficiency categories (Figure 8), and qualitative findings are organized by research questions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 discusses painpoints and challenges, including instances where alignment scores did not match participant expectations, LLM-as-judge trust issues, and latency concerns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that P1 found the system complex, P2 found priority tags arbitrarily defined, P8 preferred CLI over UI, and participants identified cases where CoPrompter's scores didn't match their judgment."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CoPrompter 'improves the ability to identify and refine instruction alignment' and 'helps in clarifying their own requirements.' These are supported by qualitative findings in Section 7, though evidence is qualitative not quantitative."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CoPrompter 'improves' alignment identification and helps users refine prompts, but the study design (no control group, no randomization, qualitative comparison to recalled past experience) is insufficient for causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'User-Centric Evaluation' broadly but the study is limited to 8 participants from a single IT company (Adobe) doing content generation tasks. The paper does not explicitly bound generalization to this narrow setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. Positive results could be due to novelty effect, demand characteristics, or the structured task format rather than the tool itself."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'GPT-4o with temperature 0' for LLM_CGen and LLM_Eval (Section 5.2) but does not specify the exact model version/snapshot date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions prompts are 'detailed in Appendix 10.5' but the appendix text provided does not include the actual full prompt text — only references to it. The prompts for criteria generation, decomposition, and evaluation are described but not shown in full."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature 0 is specified for LLM_CGen and LLM_Eval in Section 5.2."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The system's pipeline is described in detail across Sections 5.1 and 5.2: criteria generation, decomposition into atomic instructions, update module, response generation, and alignment report card module with workflow diagrams (Figures 2, 3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from guidelines to atomic instructions to criteria questions to evaluation is documented step by step in Section 5.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8.3 'Future Work and Limitations' discusses limitations including focus study format, small sample size, and latency issues."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations are somewhat specific (mentioning sample size and focus study format) but do not address key validity threats like lack of control group, single-company recruitment, demand characteristics, or novelty effects."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It acknowledges small sample size but does not bound claims to the specific participant pool, company, or task type tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (survey responses, session recordings, transcripts, thematic codes) are made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The formative study survey distribution (Slack channels and email threads) and user evaluation procedure (Microsoft Teams sessions, 75-90 minutes, recorded) are described in Sections 3 and 6."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Formative study: survey circulated across Slack channels and email threads of prompt engineers at an IT company. User evaluation: snowball sampling from an IT company, with criteria focused on experience with long-form prompts (Sections 3, 6.1)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 describes the analysis pipeline: recordings transcribed, thematic analysis approach, codes collected and categorized over three iterations using Miro."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Authors are from Adobe but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with Adobe (and internship arrangements) are clearly listed on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors are from Adobe. The tool is designed for prompt engineers, likely relevant to Adobe's products. No statement about funder independence is made, and Adobe has a potential interest in demonstrating prompt engineering tool value."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates a tool/workflow, not a pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model capability is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of model capability is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for either the formative study or user evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite collecting data from human participants in both studies."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "For the user evaluation: 3 women and 5 men, 4 with product engineering experience and 4 with industrial research applications (Section 6.1). Formative study: 28 industry prompt engineers with experience levels described."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 states criteria: 'individuals experienced in crafting long-form prompts for models from OpenAI, Meta, and others, across various use cases.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment/control conditions. All participants used CoPrompter."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not an experimental study with conditions requiring blinding."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether any participants dropped out or whether all 8 completed the full study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions API call latency as a challenge but does not report actual API costs, tokens consumed, or wall-clock time for the pipeline."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget or total API spend is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "CoPrompter improves prompt engineers' ability to identify and refine instruction alignment over traditional methods.",
       "evidence": "Qualitative feedback from 8 participants in user evaluation (Section 7). Participants described traditional methods as 'trial and error' and stated CoPrompter provided structured misalignment identification. No quantitative comparison.",
       "supported": "weak"
     },
     {
       "claim": "CoPrompter achieves a mean SUS score of 81.25, placing it in the 'A' range of excellence.",
       "evidence": "SUS survey completed by 8 participants, mean score 81.25 reported in Section 7.4 and Figure 9, compared against industry average of 68.",
       "supported": "moderate"
     },
     {
       "claim": "Prompt engineers typically require over 10 iterations and evaluation of 40+ responses to optimize complex prompts.",
       "evidence": "Formative survey of 28 industry prompt engineers (Section 3). 14/28 reported 10+ attempts, 14/28 reported testing 40+ responses.",
       "supported": "moderate"
     },
     {
       "claim": "CoPrompter helps users clarify their own requirements through criteria decomposition.",
       "evidence": "Qualitative observations from user study: participants discovered missing requirements and refined criteria during the evaluation process (Section 7.1.1). P4 quote about criteria generation being helpful.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["qualitative", "case-study"],
   "key_findings": "CoPrompter is a user-in-the-loop tool that decomposes prompt guidelines into atomic criteria questions and generates alignment reports for LLM responses. A formative study with 28 prompt engineers revealed that complex prompts require 10+ iterations and manual inspection of 40+ responses. A user evaluation with 8 industry prompt engineers showed qualitative improvements in misalignment identification and requirement clarity, with a mean SUS score of 81.25. The study is primarily qualitative with no controlled comparison to existing tools.",
   "red_flags": [
     {
       "flag": "Very small sample size for claims made",
       "detail": "The user evaluation has only 8 participants, all from a single company (Adobe), yet the paper makes broad claims about CoPrompter's effectiveness for prompt engineers generally."
     },
     {
       "flag": "No controlled comparison",
       "detail": "CoPrompter is compared to 'traditional methods' only through participant recall of past experiences, not through a controlled study. Related tools (ChainForge, SPADE, EvalLM, EvalGen) are discussed but never compared experimentally."
     },
     {
       "flag": "Potential demand characteristics",
       "detail": "Participants were shown a demo by the research team, then asked to use the tool and provide feedback. The social desirability bias in this setting (especially with Adobe employees evaluating an Adobe research tool) is not addressed."
     },
     {
       "flag": "Company evaluating its own tool",
       "detail": "Adobe researchers built CoPrompter and recruited Adobe employees to evaluate it. This conflict of interest is not acknowledged in the paper."
     }
   ],
   "cited_papers": [
     {
       "title": "ChainForge: An open-source visual programming environment for prompt engineering",
       "authors": ["Ian Arawjo", "Priyan Vaithilingam", "Martin Wattenberg", "Elena Glassman"],
       "year": 2023,
       "relevance": "Open-source tool for prompt engineering evaluation, directly related to LLM prompt development workflows."
     },
     {
       "title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences",
       "authors": ["Shreya Shankar", "JD Zamfirescu-Pereira", "Björn Hartmann", "Aditya G Parameswaran", "Ian Arawjo"],
       "year": 2024,
       "arxiv_id": "2404.12272",
       "relevance": "Framework for human-in-the-loop LLM evaluation that CoPrompter builds upon."
     },
     {
       "title": "EvalLM: Interactive evaluation of large language model prompts on user-defined criteria",
       "authors": ["Tae Soo Kim", "Yoonjoo Lee", "Jamin Shin", "Young-Ho Kim", "Juho Kim"],
       "year": 2024,
       "relevance": "Interactive LLM evaluation tool using user-defined criteria, closely related to CoPrompter's approach."
     },
     {
       "title": "SPADE: Synthesizing assertions for large language model pipelines",
       "authors": ["Shreya Shankar", "Haotian Li", "Parth Asawa"],
       "year": 2024,
       "arxiv_id": "2401.03038",
       "relevance": "Tool for automatically generating evaluation criteria for LLM pipelines."
     },
     {
       "title": "Why Johnny Can't Prompt: How Non-AI Experts Try (and Fail) to Design LLM Prompts",
       "authors": ["J.D. Zamfirescu-Pereira", "Richmond Y. Wong", "Bjoern Hartmann", "Qian Yang"],
       "year": 2023,
       "relevance": "User study on prompt engineering challenges, directly relevant to understanding prompt engineering difficulties."
     },
     {
       "title": "Interactive AI Alignment: Specification, Process, and Evaluation Alignment",
       "authors": ["Michael Terry", "Chinmay Kulkarni", "Martin Wattenberg", "Lucas Dixon", "Meredith Ringel Morris"],
       "year": 2023,
       "relevance": "Foundational framework for human-AI alignment that CoPrompter's alignment definition builds upon."
     },
     {
       "title": "Towards Bidirectional Human-AI Alignment: A Systematic Review for Clarifications, Framework, and Future Directions",
       "authors": ["Hua Shen"],
       "year": 2024,
       "arxiv_id": "2406.09264",
       "relevance": "Systematic review of human-AI alignment concepts relevant to the survey's scope on AI alignment methodology."
     },
     {
       "title": "FollowBench: A multi-level fine-grained constraints following benchmark for large language models",
       "authors": ["Yuxin Jiang"],
       "year": 2023,
       "arxiv_id": "2310.20410",
       "relevance": "Benchmark for evaluating LLM instruction-following capability."
     },
     {
       "title": "InFoBench: Evaluating Instruction Following Ability in Large Language Models",
       "authors": ["Yiwei Qin"],
       "year": 2024,
       "arxiv_id": "2401.03601",
       "relevance": "Benchmark for LLM instruction following evaluation."
     },
     {
       "title": "Calibrating LLM-Based Evaluator",
       "authors": ["Yuxuan Liu"],
       "year": 2024,
       "relevance": "Methods for calibrating LLM-as-judge evaluators, relevant to LLM evaluation methodology."
     }
   ]
 }