main_effects_sonarqube.json (8304B)
1 { 2 "model": { 3 "values": { 4 "gemma-4-26b": { 5 "mean": 0.9, 6 "effect": 0.0995, 7 "n": 43 8 }, 9 "glm-4.5-air": { 10 "mean": 0.7476, 11 "effect": -0.0529, 12 "n": 41 13 }, 14 "glm-4.7": { 15 "mean": 0.7196, 16 "effect": -0.0808, 17 "n": 81 18 }, 19 "glm-5.1": { 20 "mean": 0.8225, 21 "effect": 0.0221, 22 "n": 123 23 }, 24 "haiku-4.5": { 25 "mean": 0.7474, 26 "effect": -0.053, 27 "n": 89 28 }, 29 "kimi-k2.5": { 30 "mean": 0.4633, 31 "effect": -0.3371, 32 "n": 3 33 }, 34 "minimax-m2.7": { 35 "mean": 0.9833, 36 "effect": 0.1829, 37 "n": 3 38 }, 39 "opus-4.6": { 40 "mean": 0.8181, 41 "effect": 0.0176, 42 "n": 52 43 }, 44 "qwen-3.6-plus": { 45 "mean": 0.9059, 46 "effect": 0.1055, 47 "n": 22 48 }, 49 "sonnet-4.6": { 50 "mean": 0.8696, 51 "effect": 0.0692, 52 "n": 53 53 } 54 }, 55 "spread": 0.52 56 }, 57 "context_noise": { 58 "values": { 59 "clean": { 60 "mean": 0.8003, 61 "effect": -0.0002, 62 "n": 477 63 }, 64 "lorem_100k": { 65 "mean": 0.7633, 66 "effect": -0.0371, 67 "n": 6 68 }, 69 "lorem_10k": { 70 "mean": 0.7733, 71 "effect": -0.0271, 72 "n": 6 73 }, 74 "lorem_1k": { 75 "mean": 0.8867, 76 "effect": 0.0862, 77 "n": 3 78 }, 79 "lorem_50k": { 80 "mean": 0.735, 81 "effect": -0.0655, 82 "n": 6 83 }, 84 "wikipedia_100k": { 85 "mean": 0.7967, 86 "effect": -0.0038, 87 "n": 3 88 }, 89 "wikipedia_10k": { 90 "mean": 0.7333, 91 "effect": -0.0671, 92 "n": 3 93 }, 94 "wikipedia_1k": { 95 "mean": 0.9667, 96 "effect": 0.1662, 97 "n": 3 98 }, 99 "wikipedia_50k": { 100 "mean": 0.9067, 101 "effect": 0.1062, 102 "n": 3 103 } 104 }, 105 "spread": 0.2334 106 }, 107 "renderer": { 108 "values": { 109 "canvas": { 110 "mean": 0.7214, 111 "effect": -0.079, 112 "n": 7 113 }, 114 "dom": { 115 "mean": 0.66, 116 "effect": -0.1405, 117 "n": 5 118 }, 119 "none": { 120 "mean": 0.8034, 121 "effect": 0.003, 122 "n": 487 123 }, 124 "svg": { 125 "mean": 0.7471, 126 "effect": -0.0533, 127 "n": 7 128 }, 129 "webgl": { 130 "mean": 0.8425, 131 "effect": 0.042, 132 "n": 4 133 } 134 }, 135 "spread": 0.1825 136 }, 137 "strategy": { 138 "values": { 139 "creative_validate": { 140 "mean": 0.7363, 141 "effect": -0.0642, 142 "n": 8 143 }, 144 "delegate": { 145 "mean": 0.81, 146 "effect": 0.0095, 147 "n": 7 148 }, 149 "iterate": { 150 "mean": 0.8736, 151 "effect": 0.0732, 152 "n": 11 153 }, 154 "none": { 155 "mean": 0.7921, 156 "effect": -0.0084, 157 "n": 300 158 }, 159 "plan_first": { 160 "mean": 0.86, 161 "effect": 0.0595, 162 "n": 10 163 }, 164 "review": { 165 "mean": 0.874, 166 "effect": 0.0735, 167 "n": 5 168 }, 169 "split_work": { 170 "mean": 0.872, 171 "effect": 0.0715, 172 "n": 5 173 }, 174 "use_subagents": { 175 "mean": 0.8055, 176 "effect": 0.0051, 177 "n": 164 178 } 179 }, 180 "spread": 0.1377 181 }, 182 "architecture": { 183 "values": { 184 "best_practices": { 185 "mean": 0.925, 186 "effect": 0.1245, 187 "n": 4 188 }, 189 "none": { 190 "mean": 0.7985, 191 "effect": -0.0019, 192 "n": 501 193 }, 194 "separation": { 195 "mean": 0.892, 196 "effect": 0.0915, 197 "n": 5 198 } 199 }, 200 "spread": 0.1265 201 }, 202 "language": { 203 "values": { 204 "javascript": { 205 "mean": 0.8514, 206 "effect": 0.051, 207 "n": 21 208 }, 209 "typescript": { 210 "mean": 0.7936, 211 "effect": -0.0068, 212 "n": 469 213 }, 214 "unspecified": { 215 "mean": 0.907, 216 "effect": 0.1065, 217 "n": 20 218 } 219 }, 220 "spread": 0.1134 221 }, 222 "provider": { 223 "values": { 224 "anthropic": { 225 "mean": 0.7997, 226 "effect": -0.0007, 227 "n": 194 228 }, 229 "openrouter": { 230 "mean": 0.8869, 231 "effect": 0.0865, 232 "n": 71 233 }, 234 "zai": { 235 "mean": 0.776, 236 "effect": -0.0245, 237 "n": 245 238 } 239 }, 240 "spread": 0.1109 241 }, 242 "design_guidance": { 243 "values": { 244 "none": { 245 "mean": 0.7999, 246 "effect": -0.0006, 247 "n": 500 248 }, 249 "specific": { 250 "mean": 0.78, 251 "effect": -0.0205, 252 "n": 5 253 }, 254 "vague": { 255 "mean": 0.878, 256 "effect": 0.0775, 257 "n": 5 258 } 259 }, 260 "spread": 0.098 261 }, 262 "playwright": { 263 "values": { 264 "available": { 265 "mean": 0.7946, 266 "effect": -0.0058, 267 "n": 165 268 }, 269 "instructed": { 270 "mean": 0.7145, 271 "effect": -0.0859, 272 "n": 11 273 }, 274 "off": { 275 "mean": 0.8062, 276 "effect": 0.0057, 277 "n": 334 278 } 279 }, 280 "spread": 0.0917 281 }, 282 "max_budget": { 283 "values": { 284 "high": { 285 "mean": 0.7421, 286 "effect": -0.0584, 287 "n": 24 288 }, 289 "low": { 290 "mean": 0.8033, 291 "effect": 0.0029, 292 "n": 486 293 } 294 }, 295 "spread": 0.0612 296 }, 297 "error_checking": { 298 "values": { 299 "none": { 300 "mean": 0.8008, 301 "effect": 0.0004, 302 "n": 506 303 }, 304 "self_verify": { 305 "mean": 0.75, 306 "effect": -0.0505, 307 "n": 4 308 } 309 }, 310 "spread": 0.0508 311 }, 312 "effort": { 313 "values": { 314 "high": { 315 "mean": 0.8021, 316 "effect": 0.0017, 317 "n": 491 318 }, 319 "max": { 320 "mean": 0.7568, 321 "effect": -0.0436, 322 "n": 19 323 } 324 }, 325 "spread": 0.0453 326 }, 327 "tool_glob": { 328 "values": { 329 "off": { 330 "mean": 0.7613, 331 "effect": -0.0391, 332 "n": 30 333 }, 334 "on": { 335 "mean": 0.8029, 336 "effect": 0.0024, 337 "n": 480 338 } 339 }, 340 "spread": 0.0416 341 }, 342 "tool_grep": { 343 "values": { 344 "off": { 345 "mean": 0.8365, 346 "effect": 0.036, 347 "n": 31 348 }, 349 "on": { 350 "mean": 0.7981, 351 "effect": -0.0023, 352 "n": 479 353 } 354 }, 355 "spread": 0.0384 356 }, 357 "context_file": { 358 "values": { 359 "none": { 360 "mean": 0.7991, 361 "effect": -0.0014, 362 "n": 479 363 }, 364 "provided": { 365 "mean": 0.8219, 366 "effect": 0.0215, 367 "n": 31 368 } 369 }, 370 "spread": 0.0228 371 }, 372 "web_search": { 373 "values": { 374 "off": { 375 "mean": 0.7797, 376 "effect": -0.0207, 377 "n": 36 378 }, 379 "on": { 380 "mean": 0.802, 381 "effect": 0.0016, 382 "n": 474 383 } 384 }, 385 "spread": 0.0223 386 }, 387 "tool_write": { 388 "values": { 389 "off": { 390 "mean": 0.7824, 391 "effect": -0.018, 392 "n": 33 393 }, 394 "on": { 395 "mean": 0.8017, 396 "effect": 0.0012, 397 "n": 477 398 } 399 }, 400 "spread": 0.0193 401 }, 402 "tool_edit": { 403 "values": { 404 "off": { 405 "mean": 0.8137, 406 "effect": 0.0133, 407 "n": 35 408 }, 409 "on": { 410 "mean": 0.7995, 411 "effect": -0.001, 412 "n": 475 413 } 414 }, 415 "spread": 0.0142 416 }, 417 "human_language": { 418 "values": { 419 "en": { 420 "mean": 0.8009, 421 "effect": 0.0005, 422 "n": 481 423 }, 424 "es": { 425 "mean": 0.7924, 426 "effect": -0.008, 427 "n": 29 428 } 429 }, 430 "spread": 0.0085 431 }, 432 "tool_read": { 433 "values": { 434 "off": { 435 "mean": 0.8058, 436 "effect": 0.0054, 437 "n": 31 438 }, 439 "on": { 440 "mean": 0.8001, 441 "effect": -0.0003, 442 "n": 479 443 } 444 }, 445 "spread": 0.0057 446 }, 447 "linter": { 448 "values": { 449 "off": { 450 "mean": 0.7964, 451 "effect": -0.004, 452 "n": 39 453 }, 454 "on": { 455 "mean": 0.8008, 456 "effect": 0.0003, 457 "n": 471 458 } 459 }, 460 "spread": 0.0044 461 }, 462 "prompt_style": { 463 "values": { 464 "detailed": { 465 "mean": 0.798, 466 "effect": -0.0025, 467 "n": 30 468 }, 469 "simple": { 470 "mean": 0.8006, 471 "effect": 0.0002, 472 "n": 480 473 } 474 }, 475 "spread": 0.0026 476 } 477 }