{ "benchmark_version": "1.0.0-pilot", "edition": "2026-08-13", "scope": "pilot-evaluations", "repetitions": 1, "transport_attempt_count": 216, "response_count": 216, "request_failure_count": 0, "total_cost": 0.8526301749999993, "models": { "holo3-122b-a10b": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "layout_defect": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 0.5833333333333334, "recall": 0.16666666666666666, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0, "meanLocalizationIoU": 0, "schemaValidRate": 0.6666666666666666 }, "attempts": [ { "attempt_id": "23dbea438273d8c60080705a", "model": "holo3-122b-a10b", "case_id": "commerce-catalog-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 1873.75, "cost": 0.000623, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "98eedfb60592ddd767661484", "model": "holo3-122b-a10b", "case_id": "commerce-catalog-desktop/defect/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 4552.994, "cost": 0.00176, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "af9680af2a2b30e1cfe62d2a", "model": "holo3-122b-a10b", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 1418.961, "cost": 0.00062, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c0b53ef3936728b3b10f6408", "model": "holo3-122b-a10b", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 4570.704, "cost": 0.00176, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "29cd939395d0b0c8a9ec457e", "model": "holo3-122b-a10b", "case_id": "iot-control-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 1475.156, "cost": 0.000629, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ed5f82ba0bfafcdedf064486", "model": "holo3-122b-a10b", "case_id": "iot-control-desktop/defect/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 4420.451, "cost": 0.00176, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "b4ed337c5b0dfb848a40e741", "model": "holo3-122b-a10b", "case_id": "banking-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 1328.103, "cost": 0.0002498, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "945a7ee6b2d3b911c2d65106", "model": "holo3-122b-a10b", "case_id": "banking-ios/defect/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 4317.047, "cost": 0.0013808000000000002, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "c9837f269d3d418d5b357479", "model": "holo3-122b-a10b", "case_id": "flight-pass-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 1217.647, "cost": 0.0002408, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f82d7b078a89be5e2cbc0cff", "model": "holo3-122b-a10b", "case_id": "flight-pass-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 1689.261, "cost": 0.0003788, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e35dde7f4cec515799722e51", "model": "holo3-122b-a10b", "case_id": "smart-home-android/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 1282.939, "cost": 0.0002498, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "73419010cf1d718544eb27d5", "model": "holo3-122b-a10b", "case_id": "smart-home-android/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 1170.958, "cost": 0.0002498, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true } ] }, "element_presence": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "82f52aee9a6318687b644fff", "model": "holo3-122b-a10b", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 1409.851, "cost": 0.0005532, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d375a82286cfdcdf35596afc", "model": "holo3-122b-a10b", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 1522.357, "cost": 0.0005536, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9a3a6ca6ea142285590ab1af", "model": "holo3-122b-a10b", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 1434.018, "cost": 0.0005532, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b872d094269b8cd9a50e5d07", "model": "holo3-122b-a10b", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 1302.458, "cost": 0.0005532, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b35c845ad6e490f874777e08", "model": "holo3-122b-a10b", "case_id": "iot-control-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 1376.836, "cost": 0.0005532, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "434bc36bb13c248923fab61d", "model": "holo3-122b-a10b", "case_id": "iot-control-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 1599.989, "cost": 0.0005532, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0d804b64c43859ba20a4abc8", "model": "holo3-122b-a10b", "case_id": "banking-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 1321.134, "cost": 0.0001736, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "21f2b498a9c5025fd59d039f", "model": "holo3-122b-a10b", "case_id": "banking-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 1099.849, "cost": 0.000174, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "937711a8e1100792088da246", "model": "holo3-122b-a10b", "case_id": "flight-pass-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 1074.002, "cost": 0.000174, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b63bc44fc862ae5a1eab0c2a", "model": "holo3-122b-a10b", "case_id": "flight-pass-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 1109.622, "cost": 0.000174, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4e15c6acf5384d53609344b1", "model": "holo3-122b-a10b", "case_id": "smart-home-android/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 1108.179, "cost": 0.000174, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e3618fb2f10ed143f74e5e32", "model": "holo3-122b-a10b", "case_id": "smart-home-android/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 1110.242, "cost": 0.00017439999999999998, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "operations": { "latency_p50_ms": 1324.6185, "latency_p95_ms": 1698.4854500000001, "mean_latency_ms": 1346.2655999999997, "total_cost": 0.014265400000000001 } }, "openai/gpt-5.6-luna": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "layout_defect": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 0.8333333333333333, "recall": 0.6666666666666666, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.5, "meanLocalizationIoU": 0.219202510648658, "schemaValidRate": 0.9166666666666666 }, "attempts": [ { "attempt_id": "f8cc5c5e56a3bb41a76a4d86", "model": "openai/gpt-5.6-luna", "case_id": "commerce-catalog-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7298.096, "cost": 0.00025285, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9315262a63d0dc567ae9d206", "model": "openai/gpt-5.6-luna", "case_id": "commerce-catalog-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9782.626, "cost": 0.00035485, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.13181144553248025, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fc3d545a2a9e30eb74f3a117", "model": "openai/gpt-5.6-luna", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8330.8, "cost": 0.00025285, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2e53f85317345dc6c7a1c352", "model": "openai/gpt-5.6-luna", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8718.106, "cost": 0.00035305, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.20541592423945443, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0d3d1d07f8e14c14f12cbae2", "model": "openai/gpt-5.6-luna", "case_id": "iot-control-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8005.939, "cost": 0.00027925, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "eddcad7dcc9916e1ddbcd892", "model": "openai/gpt-5.6-luna", "case_id": "iot-control-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9559.982, "cost": 0.00035965, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "85bfe8a16fadb5ca1bef5098", "model": "openai/gpt-5.6-luna", "case_id": "banking-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7352.379, "cost": 0.0001182, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "769a8de81eafbd54d6387e66", "model": "openai/gpt-5.6-luna", "case_id": "banking-ios/defect/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 11652.957, "cost": 0.0003144, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "d9d87126e530426b7ae0b863", "model": "openai/gpt-5.6-luna", "case_id": "flight-pass-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 4323.162, "cost": 0.0001152, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9e197a0acb3e9624515e476c", "model": "openai/gpt-5.6-luna", "case_id": "flight-pass-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9528.536, "cost": 0.0001914, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.9779876941200134, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "71dd3f60d8ffbc6f8830cef7", "model": "openai/gpt-5.6-luna", "case_id": "smart-home-android/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 6596.039, "cost": 0.0000912, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6ee5ff182a36784fa3dada05", "model": "openai/gpt-5.6-luna", "case_id": "smart-home-android/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9128.588, "cost": 0.0001836, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false } ] }, "element_presence": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "34825aaa1cec6224e7087d26", "model": "openai/gpt-5.6-luna", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7112.97, "cost": 0.0002197, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b764c1871acc14856c91a105", "model": "openai/gpt-5.6-luna", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 5481.131, "cost": 0.000219825, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "358f282fde5f403dc41c3603", "model": "openai/gpt-5.6-luna", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7526.864, "cost": 0.0002197, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5c7fdc54884428de07c4221d", "model": "openai/gpt-5.6-luna", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 7646.455, "cost": 0.0002197, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1a329a52d2ef5fab1bdbfaf4", "model": "openai/gpt-5.6-luna", "case_id": "iot-control-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8630.29, "cost": 0.0002197, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7f5c72c6ff0d82536092de47", "model": "openai/gpt-5.6-luna", "case_id": "iot-control-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 7668.405, "cost": 0.0002197, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "49b91d24cf7b6078e54eabb9", "model": "openai/gpt-5.6-luna", "case_id": "banking-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8112.903, "cost": 0.0000629, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bc6ab462dfbbfb00928a27fd", "model": "openai/gpt-5.6-luna", "case_id": "banking-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 7271.038, "cost": 0.000063, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "16b9bdce56ecf883904e5384", "model": "openai/gpt-5.6-luna", "case_id": "flight-pass-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7121.389, "cost": 0.000063, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5cd9c12d94d9aaabc936d1f1", "model": "openai/gpt-5.6-luna", "case_id": "flight-pass-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 6330.061, "cost": 0.000063, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fd0e8cfcc380c6695b4b2d9b", "model": "openai/gpt-5.6-luna", "case_id": "smart-home-android/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8284.082, "cost": 0.000063, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5dd9f776499192d285ff1ed9", "model": "openai/gpt-5.6-luna", "case_id": "smart-home-android/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8260.611, "cost": 0.0000631, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "operations": { "latency_p50_ms": 7668.405, "latency_p95_ms": 9556.8374, "mean_latency_ms": 7742.193565217391, "total_cost": 0.0045628249999999995 } }, "openai/gpt-5.4": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "layout_defect": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 0.9166666666666667, "recall": 0.8333333333333334, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.6666666666666666, "meanLocalizationIoU": 0.13327797281219558, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "5dda3f38acfba8ed470aea11", "model": "openai/gpt-5.4", "case_id": "commerce-catalog-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7301.393, "cost": 0.0055625, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4f9d5c6b471ec3f12152c00e", "model": "openai/gpt-5.4", "case_id": "commerce-catalog-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9238.64, "cost": 0.0067625, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.4140664588641983, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "19e5cdf799b61e0ab960e60f", "model": "openai/gpt-5.4", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8272.883, "cost": 0.0054425, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c35632b52a83cb3bfc9f29d7", "model": "openai/gpt-5.4", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 11760.102, "cost": 0.0077525, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2190424893564988, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "61f2ad8823ba8fb7168e922f", "model": "openai/gpt-5.4", "case_id": "iot-control-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9037.365, "cost": 0.0057725, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6dba9ec0d3e6b368d406fd44", "model": "openai/gpt-5.4", "case_id": "iot-control-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9934.977, "cost": 0.0073175, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.078287237373925, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "18f8dfeb518b7f01aaa76a6f", "model": "openai/gpt-5.4", "case_id": "banking-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7071.076, "cost": 0.003195, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5b4fd679dd4ffc113a32d583", "model": "openai/gpt-5.4", "case_id": "banking-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7748.332, "cost": 0.004155, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.08827165127855152, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "62d0cd89b0f64e6a8e124fc7", "model": "openai/gpt-5.4", "case_id": "flight-pass-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7200.492, "cost": 0.002895, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0cdd24e0b8f6c6e7c71f7e4b", "model": "openai/gpt-5.4", "case_id": "flight-pass-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 12153.954, "cost": 0.007695, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e004b8f680ac06a585d464bf", "model": "openai/gpt-5.4", "case_id": "smart-home-android/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9202.109, "cost": 0.002625, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "275dd6d318cd7774c54a4a9b", "model": "openai/gpt-5.4", "case_id": "smart-home-android/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9301.874, "cost": 0.003315, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true } ] }, "element_presence": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "eea7c5fda441bd436808dd08", "model": "openai/gpt-5.4", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8700.934, "cost": 0.0044375, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "07ad888e3e7253984f74455f", "model": "openai/gpt-5.4", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 4591.945, "cost": 0.004785, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1c31d8fdb59e256f6f9cc860", "model": "openai/gpt-5.4", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8174.375, "cost": 0.0047675, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e19942a3670affb2beac5fac", "model": "openai/gpt-5.4", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 7281.739, "cost": 0.0046775, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "69c29ca06f3d89d3e2a48fb4", "model": "openai/gpt-5.4", "case_id": "iot-control-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 4810.253, "cost": 0.0044375, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6b6a9b3cc3d33723d7db03f2", "model": "openai/gpt-5.4", "case_id": "iot-control-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 6925.881, "cost": 0.0048875, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bfede36450c2a5fed0d705e5", "model": "openai/gpt-5.4", "case_id": "banking-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8255.532, "cost": 0.0018125, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "14b4ab752b925e35f2302eed", "model": "openai/gpt-5.4", "case_id": "banking-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 7998.913, "cost": 0.001965, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c6cdea4c28d15bc5687e7a39", "model": "openai/gpt-5.4", "case_id": "flight-pass-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7684.661, "cost": 0.001575, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "22ae65009b8b5c3e7ba877f1", "model": "openai/gpt-5.4", "case_id": "flight-pass-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 9972.044, "cost": 0.001815, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0dbc0b0fb0bacfea6eaed75c", "model": "openai/gpt-5.4", "case_id": "smart-home-android/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8331.486, "cost": 0.001575, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c0574e413700de2344729f33", "model": "openai/gpt-5.4", "case_id": "smart-home-android/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 7911.794, "cost": 0.0021475, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "operations": { "latency_p50_ms": 8214.9535, "latency_p95_ms": 11491.893299999998, "mean_latency_ms": 8285.948083333333, "total_cost": 0.10137249999999998 } }, "google/gemini-3.6-flash": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "layout_defect": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 0.75, "recall": 0.8333333333333334, "specificity": 0.6666666666666666, "falsePositiveRate": 0.33333333333333337, "categoryAccuracy": 0.6666666666666666, "meanLocalizationIoU": 0.2223137768622411, "schemaValidRate": 0.75 }, "attempts": [ { "attempt_id": "34e87b10ae241ca13200199f", "model": "google/gemini-3.6-flash", "case_id": "commerce-catalog-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7308.114, "cost": 0.0025965, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "79c3cc048b654e14ab9e6fdc", "model": "google/gemini-3.6-flash", "case_id": "commerce-catalog-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9657.513, "cost": 0.0043365, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.20590071454340703, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9ddd651309f8eee78f42ff5f", "model": "google/gemini-3.6-flash", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 6056.023, "cost": 0.004809, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "5a8537c8615516fe39918d2d", "model": "google/gemini-3.6-flash", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 9062.234, "cost": 0.004719, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "36e338f3f3ee50886d4d2e16", "model": "google/gemini-3.6-flash", "case_id": "iot-control-desktop/clean/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 8942.137, "cost": 0.0047265, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "453294dab965d412a6322695", "model": "google/gemini-3.6-flash", "case_id": "iot-control-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9320.95, "cost": 0.003174, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.078287237373925, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ebe861bdb53a1036f1e98434", "model": "google/gemini-3.6-flash", "case_id": "banking-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7251.691, "cost": 0.0028455, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "af8deffb570e4e1776742500", "model": "google/gemini-3.6-flash", "case_id": "banking-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8585.545, "cost": 0.004503, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.0657051964418146, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f6948db6e2359d841e9f12bf", "model": "google/gemini-3.6-flash", "case_id": "flight-pass-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 6411.38, "cost": 0.0023805, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f9d9ef3217847cc13cd11d3a", "model": "google/gemini-3.6-flash", "case_id": "flight-pass-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9482.401, "cost": 0.004638, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.9839895128143, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "03faf8feb701364f33b493c3", "model": "google/gemini-3.6-flash", "case_id": "smart-home-android/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9457.611, "cost": 0.0028455, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "537e6e849f2b5667c05c8c80", "model": "google/gemini-3.6-flash", "case_id": "smart-home-android/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9583.21, "cost": 0.0044355, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false } ] }, "element_presence": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "dd769b62c22493395d2d4baa", "model": "google/gemini-3.6-flash", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8254.57, "cost": 0.002163, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "37f6948c22c3e0365ff9e276", "model": "google/gemini-3.6-flash", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 5011.479, "cost": 0.0027945, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "284d76274e5a8460208e1707", "model": "google/gemini-3.6-flash", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7915.806, "cost": 0.002253, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5bec835cd6a58245c804a3f2", "model": "google/gemini-3.6-flash", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8310.038, "cost": 0.0025755, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "db1c87215a301699f6cb5242", "model": "google/gemini-3.6-flash", "case_id": "iot-control-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 9200.898, "cost": 0.0021555, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ff9093973d24d90783027d05", "model": "google/gemini-3.6-flash", "case_id": "iot-control-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 6307.315, "cost": 0.0025005, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "72b9b940e03431814ff03cf5", "model": "google/gemini-3.6-flash", "case_id": "banking-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8327.333, "cost": 0.0020805, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f80a9bc3aa49b3e8b4965173", "model": "google/gemini-3.6-flash", "case_id": "banking-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8036.131, "cost": 0.002547, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "80a073573fa7f0e0ddc0766c", "model": "google/gemini-3.6-flash", "case_id": "flight-pass-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 6078.112, "cost": 0.001752, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bf07578350636091e2947f98", "model": "google/gemini-3.6-flash", "case_id": "flight-pass-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8841.406, "cost": 0.002517, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "beb3900301c227f0e27ec3c3", "model": "google/gemini-3.6-flash", "case_id": "smart-home-android/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8834.448, "cost": 0.002172, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b43b887d73ea510971727f37", "model": "google/gemini-3.6-flash", "case_id": "smart-home-android/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8690.337, "cost": 0.002601, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "operations": { "latency_p50_ms": 8327.333, "latency_p95_ms": 9583.21, "mean_latency_ms": 8136.489904761905, "total_cost": 0.07412099999999999 } }, "anthropic/claude-opus-5": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "layout_defect": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 0.8333333333333333, "recall": 0.6666666666666666, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.6666666666666666, "meanLocalizationIoU": 0.053348574541154846, "schemaValidRate": 0.8333333333333334 }, "attempts": [ { "attempt_id": "a6dfc302cc1f1e63384113af", "model": "anthropic/claude-opus-5", "case_id": "commerce-catalog-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8640.232, "cost": 0.01396, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "45a436303fb3e818c2357e1f", "model": "anthropic/claude-opus-5", "case_id": "commerce-catalog-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 11455.621, "cost": 0.015935, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.16942182810769355, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6d93b14400feb757d679cab2", "model": "anthropic/claude-opus-5", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8913.445, "cost": 0.01396, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fe210fc271998c8a6166adb3", "model": "anthropic/claude-opus-5", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 11068.927, "cost": 0.015285, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "131cf1055fcfe74b7bd638c7", "model": "anthropic/claude-opus-5", "case_id": "iot-control-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9985.665, "cost": 0.01396, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dd02679c3c6eac323668c9eb", "model": "anthropic/claude-opus-5", "case_id": "iot-control-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8622.326, "cost": 0.01546, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.08138298218650988, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5f23de78283205381241a3d7", "model": "anthropic/claude-opus-5", "case_id": "banking-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9865.904, "cost": 0.007525, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e47d1b27a08f21fd2ea4bfca", "model": "anthropic/claude-opus-5", "case_id": "banking-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10405.002, "cost": 0.00885, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.06928663695272562, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "271146782886eef39c06d51d", "model": "anthropic/claude-opus-5", "case_id": "flight-pass-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8973.582, "cost": 0.007525, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6047c7c882a3e573aecfdc00", "model": "anthropic/claude-opus-5", "case_id": "flight-pass-ios/defect/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 12337.571, "cost": 0.0089, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "3b3e52c7842066b934f642f8", "model": "anthropic/claude-opus-5", "case_id": "smart-home-android/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10636.102, "cost": 0.007525, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e373213baee0b23257b80714", "model": "anthropic/claude-opus-5", "case_id": "smart-home-android/defect/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 10922.257, "cost": 0.00915, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true } ] }, "element_presence": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "7da42aa73289eff4010b4b06", "model": "anthropic/claude-opus-5", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 9809.575, "cost": 0.010675, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "07a8bda22378b8e9e960e14a", "model": "anthropic/claude-opus-5", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8167.925, "cost": 0.01069, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "538cd973558b3dd29c40d9a3", "model": "anthropic/claude-opus-5", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 9464.25, "cost": 0.01069, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "12f6971b263539a37c8049d5", "model": "anthropic/claude-opus-5", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 9885.499, "cost": 0.010685, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4dbb5eae4167fe394e0bbb14", "model": "anthropic/claude-opus-5", "case_id": "iot-control-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8356.66, "cost": 0.01068, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c158ac432f91a843dc593fad", "model": "anthropic/claude-opus-5", "case_id": "iot-control-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 9739.201, "cost": 0.01069, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "25952f78ef62e01a0da82e3b", "model": "anthropic/claude-opus-5", "case_id": "banking-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 9925.548, "cost": 0.00427, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "643eddbf60607e23d15a131f", "model": "anthropic/claude-opus-5", "case_id": "banking-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 9991.768, "cost": 0.004275, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f147992c92e2fb4689dd8dc6", "model": "anthropic/claude-opus-5", "case_id": "flight-pass-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 10174.298, "cost": 0.004265, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9f86dfc054064006faf0f37d", "model": "anthropic/claude-opus-5", "case_id": "flight-pass-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 11074.312, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "890e3a1f0fea53b5b8cd61c4", "model": "anthropic/claude-opus-5", "case_id": "smart-home-android/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 9521.605, "cost": 0.004265, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "451d92e94e14c6cbb8c56ac9", "model": "anthropic/claude-opus-5", "case_id": "smart-home-android/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 12069.287, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "operations": { "latency_p50_ms": 9875.7015, "latency_p95_ms": 11436.55555, "mean_latency_ms": 9852.124272727275, "total_cost": 0.22777999999999998 } }, "x-ai/grok-4.5": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "layout_defect": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 1, "recall": 1, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.6666666666666666, "meanLocalizationIoU": 0.20701244492353535, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "9684eae2ae921a2137022d03", "model": "x-ai/grok-4.5", "case_id": "commerce-catalog-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8667.634, "cost": 0.004074, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f73909e6e51af755618010c8", "model": "x-ai/grok-4.5", "case_id": "commerce-catalog-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8632.286, "cost": 0.0040904, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7c36cf9c68baff7251c0b08f", "model": "x-ai/grok-4.5", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 6673.149, "cost": 0.0038384, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "53803a2cda26fed087b9d034", "model": "x-ai/grok-4.5", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7653.396, "cost": 0.0040664, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2849715735723838, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c7c6336fdc47dbc4ba3b3a57", "model": "x-ai/grok-4.5", "case_id": "iot-control-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7772.557, "cost": 0.0038384, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8c0e83283af1bcbd05155f15", "model": "x-ai/grok-4.5", "case_id": "iot-control-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8655.981, "cost": 0.0040604, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.0744680882752378, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1a54eb0d19be7ca1402abd5b", "model": "x-ai/grok-4.5", "case_id": "banking-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7073.121, "cost": 0.0012776, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "30b925069f52475d9597a8f9", "model": "x-ai/grok-4.5", "case_id": "banking-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 6177.766, "cost": 0.0021584, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.191231137859705, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8c7cc0b84e0be1e78d2ab4b9", "model": "x-ai/grok-4.5", "case_id": "flight-pass-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7793.393, "cost": 0.0012776, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8a061fee728ccf33c2ed72e5", "model": "x-ai/grok-4.5", "case_id": "flight-pass-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 6350.449, "cost": 0.0014336, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f3e119969d8f636c75a2ee44", "model": "x-ai/grok-4.5", "case_id": "smart-home-android/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7817.824, "cost": 0.0012776, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ab31d0cace0338940018d643", "model": "x-ai/grok-4.5", "case_id": "smart-home-android/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 5505.525, "cost": 0.0015656, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.6914038698338856, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false } ] }, "element_presence": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "d2760d9f298681f58ae12ac9", "model": "x-ai/grok-4.5", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7726.759, "cost": 0.0031884, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "03f1a457b90630015d276253", "model": "x-ai/grok-4.5", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 6260.456, "cost": 0.0031904, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "08588bcdfd8ca5a6069b463b", "model": "x-ai/grok-4.5", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7430.204, "cost": 0.0031884, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d7f196a91be605afe4cfd4f2", "model": "x-ai/grok-4.5", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 7623.849, "cost": 0.0031884, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "90ce84aaff541a48712b4203", "model": "x-ai/grok-4.5", "case_id": "iot-control-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7425.814, "cost": 0.0031884, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b53cc1f8ca7a9790fe238632", "model": "x-ai/grok-4.5", "case_id": "iot-control-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 4719.088, "cost": 0.0031884, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "17af2215037821cce8e7dfe7", "model": "x-ai/grok-4.5", "case_id": "banking-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8915.689, "cost": 0.0012784, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "713c91a6dc59e9bc8b367538", "model": "x-ai/grok-4.5", "case_id": "banking-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8453.132, "cost": 0.0012804, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f67e384d87d30b555581110e", "model": "x-ai/grok-4.5", "case_id": "flight-pass-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7409.033, "cost": 0.0012804, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "67bd5af6b7f5683529598531", "model": "x-ai/grok-4.5", "case_id": "flight-pass-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8266.412, "cost": 0.0012804, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6df6037a1a61f2adcdac0c8d", "model": "x-ai/grok-4.5", "case_id": "smart-home-android/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7821.943, "cost": 0.0012804, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d7049a89854b893c9962127e", "model": "x-ai/grok-4.5", "case_id": "smart-home-android/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 7994.068, "cost": 0.0012824, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "operations": { "latency_p50_ms": 7690.077499999999, "latency_p95_ms": 8665.88605, "mean_latency_ms": 7450.813666666666, "total_cost": 0.0597732 } }, "anthropic/claude-sonnet-5": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "layout_defect": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 0.8333333333333334, "recall": 0.8333333333333334, "specificity": 0.8333333333333334, "falsePositiveRate": 0.16666666666666663, "categoryAccuracy": 0.6666666666666666, "meanLocalizationIoU": 0.07062669728909692, "schemaValidRate": 0.9166666666666666 }, "attempts": [ { "attempt_id": "51892af080675f38c33b555b", "model": "anthropic/claude-sonnet-5", "case_id": "commerce-catalog-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 12126.283, "cost": 0.005934, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3af99d831d2b99bfbabdc19a", "model": "anthropic/claude-sonnet-5", "case_id": "commerce-catalog-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 11343.544, "cost": 0.006174, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.22458948111819296, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c6a44b560690cdb8d73b58d1", "model": "anthropic/claude-sonnet-5", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8670.418, "cost": 0.005964, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "cbc4564c4ccdc819e5d63cfb", "model": "anthropic/claude-sonnet-5", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 11243.435, "cost": 0.006134, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.1282066210795356, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "203ec760c6140d3cbbb8e12f", "model": "anthropic/claude-sonnet-5", "case_id": "iot-control-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9569.148, "cost": 0.005934, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "718ad1e68ce290c1887004a5", "model": "anthropic/claude-sonnet-5", "case_id": "iot-control-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10760.483, "cost": 0.006224, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07096408153685295, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "14975f1c86cf7695e0f36dd3", "model": "anthropic/claude-sonnet-5", "case_id": "banking-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10247.324, "cost": 0.00354, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "1316605dc2d2dcddee0d2b74", "model": "anthropic/claude-sonnet-5", "case_id": "banking-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10577.129, "cost": 0.00357, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d3fef3ffc4238e01546b08de", "model": "anthropic/claude-sonnet-5", "case_id": "flight-pass-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10538.047, "cost": 0.00303, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "aafe9a5e8c9f8da7d3cc5212", "model": "anthropic/claude-sonnet-5", "case_id": "flight-pass-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10046.405, "cost": 0.00354, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0f0d281d9bb5b615d03543f5", "model": "anthropic/claude-sonnet-5", "case_id": "smart-home-android/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10275.663, "cost": 0.00309, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "cb4f93c56144bb3369801136", "model": "anthropic/claude-sonnet-5", "case_id": "smart-home-android/defect/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 10868.543, "cost": 0.00375, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true } ] }, "element_presence": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "cb60feac6b4ba3afcbbe94cc", "model": "anthropic/claude-sonnet-5", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 6599.364, "cost": 0.00427, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "eb319fd25dbecb5aa14e6a35", "model": "anthropic/claude-sonnet-5", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 7768.595, "cost": 0.004276, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c60c499a69037b0356a0394d", "model": "anthropic/claude-sonnet-5", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 9993.651, "cost": 0.004276, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "285c94357f03739dc0cad665", "model": "anthropic/claude-sonnet-5", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 12364.155, "cost": 0.004274, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "861ad8a9242eadc11904a4bb", "model": "anthropic/claude-sonnet-5", "case_id": "iot-control-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 10463.62, "cost": 0.004272, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1a124269ad06ef1fb54cf068", "model": "anthropic/claude-sonnet-5", "case_id": "iot-control-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 9327.416, "cost": 0.004276, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b1c1f5f4f9b9aa5afe03c6a7", "model": "anthropic/claude-sonnet-5", "case_id": "banking-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 10321.208, "cost": 0.001708, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f1f4a0244fa48e7d6cbddadb", "model": "anthropic/claude-sonnet-5", "case_id": "banking-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 6226.777, "cost": 0.00171, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bb5fbf1764dc2abe2597b5ff", "model": "anthropic/claude-sonnet-5", "case_id": "flight-pass-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 10023.742, "cost": 0.001706, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f245091f154ec5db50e97992", "model": "anthropic/claude-sonnet-5", "case_id": "flight-pass-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 10540.43, "cost": 0.001712, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a1c87fee91c6585e0d8d0214", "model": "anthropic/claude-sonnet-5", "case_id": "smart-home-android/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8790.522, "cost": 0.001706, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "25c18e4bf34eb7cf5debe5ce", "model": "anthropic/claude-sonnet-5", "case_id": "smart-home-android/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8847.319, "cost": 0.001712, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "operations": { "latency_p50_ms": 10247.324, "latency_p95_ms": 12048.0091, "mean_latency_ms": 9854.985999999999, "total_cost": 0.09278200000000002 } }, "openai/gpt-5.6-sol": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "layout_defect": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 0.9166666666666667, "recall": 0.8333333333333334, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.6666666666666666, "meanLocalizationIoU": 0.13597855676708906, "schemaValidRate": 0.9166666666666666 }, "attempts": [ { "attempt_id": "7f1f1b58d15f3d8457cf94ef", "model": "openai/gpt-5.6-sol", "case_id": "commerce-catalog-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 26990.258, "cost": 0.0128825, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d57234eb05a8495e4d166545", "model": "openai/gpt-5.6-sol", "case_id": "commerce-catalog-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 15883.207, "cost": 0.0183125, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.21339640230006862, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1872a35c3f6e62c408bd4263", "model": "openai/gpt-5.6-sol", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8530.296, "cost": 0.0127625, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a5623af528986deaadba7c5d", "model": "openai/gpt-5.6-sol", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 12683.176, "cost": 0.0180125, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.16538528823397272, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "db0ef4151ec2f4d617f2c35f", "model": "openai/gpt-5.6-sol", "case_id": "iot-control-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7260.095, "cost": 0.0127625, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dd69b131aadc3ddc33c5e026", "model": "openai/gpt-5.6-sol", "case_id": "iot-control-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9010.398, "cost": 0.0152525, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07793617352919888, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "db14c2a3e48a43ac058545e9", "model": "openai/gpt-5.6-sol", "case_id": "banking-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8289.187, "cost": 0.00717, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4d6a7c6c3adfc139854a4e8a", "model": "openai/gpt-5.6-sol", "case_id": "banking-ios/defect/layout-defect", "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 16929.268, "cost": null, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "0f89ca408a0e1d30b3d0db07", "model": "openai/gpt-5.6-sol", "case_id": "flight-pass-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 7858.907, "cost": 0.00549, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "23dee18e03a0787a00d7136a", "model": "openai/gpt-5.6-sol", "case_id": "flight-pass-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10065.858, "cost": 0.00873, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.3591534765392941, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b081865995a6042e60ea52e7", "model": "openai/gpt-5.6-sol", "case_id": "smart-home-android/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 8109.62, "cost": 0.00468, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "30cb4837a2e98e07ad5272b1", "model": "openai/gpt-5.6-sol", "case_id": "smart-home-android/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 11319.316, "cost": 0.01224, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false } ] }, "element_presence": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "137cef6f524cc9ed9259773a", "model": "openai/gpt-5.6-sol", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 7959.302, "cost": 0.010985, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9c896fd99a14dcc3ab8d8f5c", "model": "openai/gpt-5.6-sol", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 9134.733, "cost": 0.01099125, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7c60507ca04920d6c79df659", "model": "openai/gpt-5.6-sol", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8530.849, "cost": 0.010985, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "411c7b7115a7035fde6474bf", "model": "openai/gpt-5.6-sol", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 7123.589, "cost": 0.010985, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b060361ecd6aa258ea4b9d0d", "model": "openai/gpt-5.6-sol", "case_id": "iot-control-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 5414.139, "cost": 0.010985, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0dec76a5e4fb03bf07766769", "model": "openai/gpt-5.6-sol", "case_id": "iot-control-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8099.473, "cost": 0.010985, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f5514620b351f99dfa073996", "model": "openai/gpt-5.6-sol", "case_id": "banking-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 6423.032, "cost": 0.003145, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4dc6e8a0048e81a3037eadbb", "model": "openai/gpt-5.6-sol", "case_id": "banking-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 5694.539, "cost": 0.00315, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d68fcfc3e5077385496e0445", "model": "openai/gpt-5.6-sol", "case_id": "flight-pass-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8960.142, "cost": 0.00315, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "82ac7c6eba210fcd6277b923", "model": "openai/gpt-5.6-sol", "case_id": "flight-pass-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 9159.443, "cost": 0.00315, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b9c98c68c64ea8bc0404a2bc", "model": "openai/gpt-5.6-sol", "case_id": "smart-home-android/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 8183.404, "cost": 0.00315, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0b85112b02652bab0859b0e1", "model": "openai/gpt-5.6-sol", "case_id": "smart-home-android/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 8846.465, "cost": 0.003155, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "operations": { "latency_p50_ms": 8530.296, "latency_p95_ms": 15563.203899999997, "mean_latency_ms": 9544.757739130435, "total_cost": 0.21311124999999995 } }, "google/gemini-3.1-pro-preview": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "layout_defect": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 0.9166666666666667, "recall": 1, "specificity": 0.8333333333333334, "falsePositiveRate": 0.16666666666666663, "categoryAccuracy": 0.6666666666666666, "meanLocalizationIoU": 0.1141406732420196, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "dbb1224c6a8603010693b457", "model": "google/gemini-3.1-pro-preview", "case_id": "commerce-catalog-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 31207.038, "cost": 0.002896, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "82c16d3ab1cfd7c0abd8aef1", "model": "google/gemini-3.1-pro-preview", "case_id": "commerce-catalog-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 24132.01, "cost": 0.003184, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.21205210926928125, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8c5577363deccef80939306f", "model": "google/gemini-3.1-pro-preview", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 43125.249, "cost": 0.002764, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f6fac02b56376c21363a7caa", "model": "google/gemini-3.1-pro-preview", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 32339.94, "cost": 0.003184, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3155a15cb6cffac17167af18", "model": "google/gemini-3.1-pro-preview", "case_id": "iot-control-desktop/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 22268.843, "cost": 0.00274, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ff3ace4c36db5b4fe0cb24a7", "model": "google/gemini-3.1-pro-preview", "case_id": "iot-control-desktop/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 37136.277, "cost": 0.003232, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07710106711068371, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "96f93dadbc257bb929e52f38", "model": "google/gemini-3.1-pro-preview", "case_id": "banking-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 11053.616, "cost": 0.003072, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "4cc51f2bb24aa42571e6d1d6", "model": "google/gemini-3.1-pro-preview", "case_id": "banking-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 9759.045, "cost": 0.003204, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2321704154885678, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e699b175f65bf04d4cf0c215", "model": "google/gemini-3.1-pro-preview", "case_id": "flight-pass-ios/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10479.734, "cost": 0.002724, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3502e9ec3615d01afa2cd779", "model": "google/gemini-3.1-pro-preview", "case_id": "flight-pass-ios/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 17761.134, "cost": 0.003336, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c900e421a6e69566375c9a00", "model": "google/gemini-3.1-pro-preview", "case_id": "smart-home-android/clean/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 11714.36, "cost": 0.002844, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "36c734f69993d86c86059751", "model": "google/gemini-3.1-pro-preview", "case_id": "smart-home-android/defect/layout-defect", "track": "layout-defect", "status": "success", "wall_time_ms": 10053.838, "cost": 0.00336, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.1635204475835848, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false } ] }, "element_presence": { "transport_attempt_count": 12, "request_failure_count": 0, "metrics": { "balancedAccuracy": 0.9166666666666667, "presentRecall": 1, "absentSpecificity": 0.8333333333333334, "falsePositiveRate": 0.16666666666666663, "schemaValidRate": 1 }, "attempts": [ { "attempt_id": "a6d412a000fb970857a79da1", "model": "google/gemini-3.1-pro-preview", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 10763.305, "cost": 0.002374, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4ace901303e1cc179815deef", "model": "google/gemini-3.1-pro-preview", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 39191.65, "cost": 0.002376, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a7c02fc1673383aaab7c549e", "model": "google/gemini-3.1-pro-preview", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 10793.531, "cost": 0.002374, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "15c41bc62cf07ec55e0e0352", "model": "google/gemini-3.1-pro-preview", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 9867.611, "cost": 0.002374, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3ba9d221cf164beb95f73990", "model": "google/gemini-3.1-pro-preview", "case_id": "iot-control-desktop/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 26308.922, "cost": 0.002374, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "61ffc71ea0f4e80b16dd35ad", "model": "google/gemini-3.1-pro-preview", "case_id": "iot-control-desktop/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 12431.461, "cost": 0.002374, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ff68133ff277ff9119fae86f", "model": "google/gemini-3.1-pro-preview", "case_id": "banking-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 10084.288, "cost": 0.002344, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dd128840d10536cc41637fe3", "model": "google/gemini-3.1-pro-preview", "case_id": "banking-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 12430.502, "cost": 0.002346, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8857bd4723e61d08a11d4f81", "model": "google/gemini-3.1-pro-preview", "case_id": "flight-pass-ios/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 9940.935, "cost": 0.002346, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "61ad6193bcc7842cbb71bdf4", "model": "google/gemini-3.1-pro-preview", "case_id": "flight-pass-ios/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 10303.591, "cost": 0.002346, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f61c77980ceda559784f8320", "model": "google/gemini-3.1-pro-preview", "case_id": "smart-home-android/clean/element-presence/present", "track": "element-presence", "status": "success", "wall_time_ms": 15902.815, "cost": 0.002346, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "74e913ac5ed35220b7eb5115", "model": "google/gemini-3.1-pro-preview", "case_id": "smart-home-android/clean/element-presence/absent", "track": "element-presence", "status": "success", "wall_time_ms": 12132.225, "cost": 0.002348, "schemaValid": true, "classificationCorrect": false, "expectedFound": false, "predictedFound": true, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false } ] }, "operations": { "latency_p50_ms": 12281.3635, "latency_p95_ms": 38883.34405, "mean_latency_ms": 18382.579999999998, "total_cost": 0.06486199999999999 } } } }