{ "edition": "2026-10-17", "benchmark_version": "1.0.0-pilot", "status": "complete", "stop_reason": null, "repetitions": 3, "planned_requests": 1368, "completed_requests": 1368, "transport_attempts": 1368, "total_cost_usd": 3.3802466090000087, "models": { "openai/gpt-6-luna": { "name": "GPT-6 Luna", "classification_only": false, "point_source": "independently predicted", "planned_requests": 144, "completed_requests": 144, "transport_attempts": 144, "request_failures": 0, "invalid_responses": 0, "resolved_models": [ "openai/gpt-6-luna" ], "grounding": { "metrics": { "pointAccuracy": 0.9861111111111112, "meanIoU": 0.7955995199626874, "iouAt50": 0.7777777777777778, "schemaValidRate": 1 }, "hits": 71, "count": 72, "consistency": [ { "case_id": "commerce-catalog-desktop/clean/grounding/search", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 1 }, { "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9873015873015873 }, { "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 1 }, { "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "hit_consistent": true, "meanPairwisePointDistance": 20.666666666666668, "meanPairwiseBoxIoU": 0.9666666666666667 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.8670017925337076 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.5480665921650649 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9644428152492669 }, { "case_id": "iot-control-desktop/clean/grounding/devices-nav", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "iot-control-desktop/clean/grounding/restart-device", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 1 }, { "case_id": "iot-control-desktop/clean/grounding/auto-mode", "hit_consistent": true, "meanPairwisePointDistance": 37.333333333333336, "meanPairwiseBoxIoU": 0.9345063538611926 }, { "case_id": "iot-control-desktop/clean/grounding/pump-card", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9916666666666667 }, { "case_id": "banking-ios/clean/grounding/profile", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "banking-ios/clean/grounding/transaction-search", "hit_consistent": true, "meanPairwisePointDistance": 6.666666666666667, "meanPairwiseBoxIoU": 1 }, { "case_id": "banking-ios/clean/grounding/hide-balance", "hit_consistent": true, "meanPairwisePointDistance": 106.00209641532776, "meanPairwiseBoxIoU": 0.9357142857142856 }, { "case_id": "banking-ios/clean/grounding/transfer", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9883040935672515 }, { "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "flight-pass-ios/clean/grounding/check-in", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "flight-pass-ios/clean/grounding/trips-tab", "hit_consistent": true, "meanPairwisePointDistance": 74.74684405808169, "meanPairwiseBoxIoU": 0.9912280701754387 }, { "case_id": "flight-pass-ios/clean/grounding/profile-tab", "hit_consistent": false, "meanPairwisePointDistance": 343.48156897253267, "meanPairwiseBoxIoU": 0.3289473684210526 }, { "case_id": "smart-home-android/clean/grounding/more", "hit_consistent": true, "meanPairwisePointDistance": 1.1380711874576983, "meanPairwiseBoxIoU": 0.9870550161812298 }, { "case_id": "smart-home-android/clean/grounding/room-search", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9743589743589745 }, { "case_id": "smart-home-android/clean/grounding/thermostat-card", "hit_consistent": true, "meanPairwisePointDistance": 204.1408016171663, "meanPairwiseBoxIoU": 0.3333333333333333 }, { "case_id": "smart-home-android/clean/grounding/add-device", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9883040935672515 } ] }, "presence": { "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "correct": 36, "count": 36 }, "defects": { "metrics": { "balancedAccuracy": 0.9166666666666667, "recall": 0.8333333333333334, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.5555555555555556, "meanLocalizationIoU": 0.1647503667678807, "schemaValidRate": 1 }, "correct": 33, "count": 36, "category_correct": 10, "category_count": 18, "region_supported": true }, "operations_by_track": { "grounding": { "count": 72, "transport_attempts": 72, "p50_ms": 2308.584, "p95_ms": 3361.4671500000004, "cost_usd": 0.009643449999999994 }, "element-presence": { "count": 36, "transport_attempts": 36, "p50_ms": 1449.7024999999999, "p95_ms": 1882.2734999999998, "cost_usd": 0.0027089050000000006 }, "layout-defect": { "count": 36, "transport_attempts": 36, "p50_ms": 2718.495, "p95_ms": 5892.71825, "cost_usd": 0.0063039199999999984 } }, "operations": { "p50_ms": 2176.6710000000003, "p95_ms": 3825.2121499999994, "cost_usd": 0.018656275, "missing_cost_count": 0 }, "attempts": [ { "attempt_id": "add3f01d2b806e061f76bfce", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3333.15, "cost": 0.000282425, "schemaValid": true, "pointHit": true, "boxIoU": 0.9675787247276455, "boxIoUAt50": true }, { "attempt_id": "691d67b40b273ef30b89ac6c", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1916.894, "cost": 0.000247925, "schemaValid": true, "pointHit": true, "boxIoU": 0.9306130169374993, "boxIoUAt50": true }, { "attempt_id": "39e2472bc435dd7176349056", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2649.6, "cost": 0.000277425, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "bb1286ba30985ea74ceffbee", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3774.74, "cost": 0.00035005, "schemaValid": true, "pointHit": true, "boxIoU": 0.3991789541161305, "boxIoUAt50": false }, { "attempt_id": "baf9ba85f61de4169b1a09d3", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2334.403, "cost": 0.00028005, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "afd985a9c6ba61e17ede4827", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3980.693, "cost": 0.00037905, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.14121017890140575, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "04d2c785daeec294a3c75390", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1545.177, "cost": 0.0002183, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a98ea50628c88b42d94e100e", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1279.822, "cost": 0.000218425, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bf3652cde2b8f67bc5ee9457", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2661.148, "cost": 0.0003153, "schemaValid": true, "pointHit": true, "boxIoU": 0.9226695881695193, "boxIoUAt50": true }, { "attempt_id": "13119eebbbb5c60c2eead109", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2423.97, "cost": 0.000283425, "schemaValid": true, "pointHit": true, "boxIoU": 0.9897563499246878, "boxIoUAt50": true }, { "attempt_id": "a65b46983516e84bb4c7b95d", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2012.566, "cost": 0.0002778, "schemaValid": true, "pointHit": true, "boxIoU": 0.9782124823785875, "boxIoUAt50": true }, { "attempt_id": "c9f3a302422f87191e9fe984", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2238.883, "cost": 0.000289425, "schemaValid": true, "pointHit": true, "boxIoU": 0.4648597652852114, "boxIoUAt50": false }, { "attempt_id": "c38c2953d3b60a802fa51e56", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 1986.797, "cost": 0.00027905, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "190e9d1bb81dc818483e6ac4", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2587.051, "cost": 0.00030555, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.20541592423945443, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c8c7eb4c8b24fbd1e0815a3f", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1571.301, "cost": 0.0002183, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2c2da6a417df08220a550864", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1571.95, "cost": 0.0002183, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1165a1dc0b46d7d19a9e6e9b", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2211.547, "cost": 0.000284925, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "61d8285c4bea997d2c767932", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1975.797, "cost": 0.000271925, "schemaValid": true, "pointHit": true, "boxIoU": 0.9722465288420963, "boxIoUAt50": true }, { "attempt_id": "92b33455d31f0616d9a89615", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2621.293, "cost": 0.000308925, "schemaValid": true, "pointHit": true, "boxIoU": 0.48585044389287935, "boxIoUAt50": false }, { "attempt_id": "a29497a8fa9ddd095d71aa5e", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2804.392, "cost": 0.0003128, "schemaValid": true, "pointHit": true, "boxIoU": 0.9637247179241175, "boxIoUAt50": true }, { "attempt_id": "7b2a505daa512617b4018c95", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2044.043, "cost": 0.00028755, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "84b84bca9e00218f2defa374", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3623.237, "cost": 0.00035455, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07557447130104133, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "36c8338737e0c6f770e1829d", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1487.41, "cost": 0.0002183, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3b2c6092e089e55a81ac2cf1", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1570.776, "cost": 0.0002183, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "80422e2a68108dbdf3cbaba7", "case_id": "banking-ios/clean/grounding/profile", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2459.148, "cost": 0.0001332, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774944177009569, "boxIoUAt50": true }, { "attempt_id": "1e6d3a87bb2351362833c010", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2814.586, "cost": 0.0001217, "schemaValid": true, "pointHit": true, "boxIoU": 0.8476522453114688, "boxIoUAt50": true }, { "attempt_id": "b29c161caf2c5b1a49f273ba", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2587.978, "cost": 0.0001472, "schemaValid": true, "pointHit": true, "boxIoU": 0.3593636887968583, "boxIoUAt50": false }, { "attempt_id": "a7a49352ac7e141469a5c114", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1872.549, "cost": 0.0000876, "schemaValid": true, "pointHit": true, "boxIoU": 0.9598488211838173, "boxIoUAt50": true }, { "attempt_id": "582196dc8c4d01f6540eb25d", "case_id": "banking-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2158.388, "cost": 0.0001274, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "40fdc5ee64f23f05c615450d", "case_id": "banking-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 6070.281, "cost": 0.0003324, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "b00fccf4d019317b4aaa3955", "case_id": "banking-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1722.227, "cost": 0.0000615, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7e5e665178a1f358eff97a50", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1243.372, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c2c569ea845705ac9d612c72", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2487.678, "cost": 0.0001245, "schemaValid": true, "pointHit": true, "boxIoU": 0.9846068132952047, "boxIoUAt50": true }, { "attempt_id": "5bc8739905fea2fbc8864c85", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2783.897, "cost": 0.0001112, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "9a96cf3be026b253c65eff04", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1629.399, "cost": 0.0000877, "schemaValid": true, "pointHit": true, "boxIoU": 0.728029984198174, "boxIoUAt50": true }, { "attempt_id": "cf0dc455b1b1b1e524741836", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2532.819, "cost": 0.0001447, "schemaValid": true, "pointHit": true, "boxIoU": 0.7185816365217428, "boxIoUAt50": true }, { "attempt_id": "7f46711e19ae56846c3825dd", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 1755.187, "cost": 0.0001199, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "94ca686fca800fd648bb88ef", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2788.67, "cost": 0.0001839, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.6250531328258307, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "29eb3264022ee8d2ca088715", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1241.086, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "efa54118fce1c3ab3b59bcaf", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1124.694, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "455004c0fd4f05505ffeefee", "case_id": "smart-home-android/clean/grounding/more", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1474.103, "cost": 0.0000878, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774944177009569, "boxIoUAt50": true }, { "attempt_id": "11a078af3c3e41ae754a19d6", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1877.275, "cost": 0.0001162, "schemaValid": true, "pointHit": true, "boxIoU": 0.8606775633691806, "boxIoUAt50": true }, { "attempt_id": "dd74c8c5de9e2f947d37f655", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1455.037, "cost": 0.0000878, "schemaValid": true, "pointHit": true, "boxIoU": 0.9638329208225912, "boxIoUAt50": true }, { "attempt_id": "2f4f88d234438474100eedc8", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1382.46, "cost": 0.0000877, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "c091e6a0004aa048ce815349", "case_id": "smart-home-android/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 1934.074, "cost": 0.0000884, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e8e64263b7cae8c1b8032e6c", "case_id": "smart-home-android/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4042.619, "cost": 0.0002364, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "52c90789cc9c15d73e09deab", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1194.699, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "63d5b49f875601b49a03be8d", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1209.837, "cost": 0.0000617, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "aed40a501a3d37badab98b91", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2621.583, "cost": 0.00007659, "schemaValid": true, "pointHit": true, "boxIoU": 0.9675787247276455, "boxIoUAt50": true }, { "attempt_id": "644fdaee6227f23bf075c655", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2122.038, "cost": 0.00003759, "schemaValid": true, "pointHit": true, "boxIoU": 0.9306130169374993, "boxIoUAt50": true }, { "attempt_id": "62babd2722e72038e4fd3103", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1544.865, "cost": 0.00003759, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "a54d168dc5204f8b87707bb9", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2706.702, "cost": 0.0000966, "schemaValid": true, "pointHit": true, "boxIoU": 0.4201883727538216, "boxIoUAt50": false }, { "attempt_id": "beb20d070797f8d7b863d858", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1879.851, "cost": 0.00005166, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4e079aba55ade029e2edfabf", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3444.085, "cost": 0.00014016, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.13536925427232455, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fad8e801822a1f1e60250d05", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 3754.84, "cost": 0.00002418, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f9d4e4b215dc9dcbe25e1d15", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1646.593, "cost": 0.00002419, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "57cd44ddec53ab9538939fe3", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3181.622, "cost": 0.00013658, "schemaValid": true, "pointHit": true, "boxIoU": 0.7959339812201163, "boxIoUAt50": true }, { "attempt_id": "5809bea7b050203c10359f74", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2894.524, "cost": 0.00006809, "schemaValid": true, "pointHit": true, "boxIoU": 0.3264616458221359, "boxIoUAt50": false }, { "attempt_id": "2b35e4d667c266507710dce2", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2214.953, "cost": 0.00007358, "schemaValid": true, "pointHit": true, "boxIoU": 0.9782124823785875, "boxIoUAt50": true }, { "attempt_id": "7f0ea138d4bcbc6b7c34e81c", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2668.835, "cost": 0.00008259, "schemaValid": true, "pointHit": true, "boxIoU": 0.4869959445845072, "boxIoUAt50": false }, { "attempt_id": "8f3ac39b03c8cb2bf0d2899b", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1906.671, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "80ccdf2ddc82bc07e6b8689c", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2648.32, "cost": 0.00011566, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.24516232025160084, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4303063a5d3c42eff282feca", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1966.248, "cost": 0.00002418, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "00d2877128f7bc40527ab1b2", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1667.859, "cost": 0.00002418, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "11e9970d4d20796665631d86", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2203.676, "cost": 0.00007459, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "c27012f2f0ce2a4e9497b2ed", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2248.357, "cost": 0.00006659, "schemaValid": true, "pointHit": true, "boxIoU": 0.9722465288420963, "boxIoUAt50": true }, { "attempt_id": "cd6af6a8b637392f01e817c1", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2860.317, "cost": 0.00009859, "schemaValid": true, "pointHit": true, "boxIoU": 0.4384239198628043, "boxIoUAt50": false }, { "attempt_id": "b72282d18b17dd4d4f510ced", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3249.525, "cost": 0.00008462, "schemaValid": true, "pointHit": true, "boxIoU": 0.9637247179241175, "boxIoUAt50": true }, { "attempt_id": "256a3dc85a1acfd1e17488d9", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1917.032, "cost": 0.00005766, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b514fbf705fd6fde92be87c6", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3143.828, "cost": 0.00012166, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07557447130104133, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e053790b2e5dc101be1d591b", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1769.592, "cost": 0.00002418, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7d205e3e5968199b59216b26", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1638.803, "cost": 0.00002418, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "410a0c6f4fa0377ddac1b023", "case_id": "banking-ios/clean/grounding/profile", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3396.077, "cost": 0.0001577, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774944177009569, "boxIoUAt50": true }, { "attempt_id": "03e1421f614f1ccfbe90e627", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2170.645, "cost": 0.0001302, "schemaValid": true, "pointHit": true, "boxIoU": 0.8476522453114688, "boxIoUAt50": true }, { "attempt_id": "1369d22a7b9bd8512bf3df2b", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2384.115, "cost": 0.0001347, "schemaValid": true, "pointHit": true, "boxIoU": 0.32515361407108945, "boxIoUAt50": false }, { "attempt_id": "60985590d249cb0e254fd11e", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1749.716, "cost": 0.0000876, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "688e6c56fe40d9d5888d2e6b", "case_id": "banking-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1868.123, "cost": 0.0001159, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "de6e68fb74d886e3b1a27ea2", "case_id": "banking-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 5974.937, "cost": 0.0003359, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8db5f28c4d55744ab1540b27", "case_id": "banking-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1171.223, "cost": 0.0000615, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "efb9aea7b117d9ffa2265dd9", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1453.389, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1d0da56dc729a4fd4add68a5", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2264.455, "cost": 0.0001185, "schemaValid": true, "pointHit": true, "boxIoU": 0.9846068132952047, "boxIoUAt50": true }, { "attempt_id": "a01aaa5ae644b30521aa204e", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1468.499, "cost": 0.0000877, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "17591881c2327946bb99b693", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1622.819, "cost": 0.0000877, "schemaValid": true, "pointHit": true, "boxIoU": 0.7186492442109241, "boxIoUAt50": true }, { "attempt_id": "8d54db17c4d8f06830321f1f", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1546.078, "cost": 0.0000877, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "db3ed4f3006cbc6ede544464", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2198.864, "cost": 0.0001224, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1e3a7367dc1c2727f732c54d", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3675.722, "cost": 0.0002074, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.6250531328258307, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "027af6b624ee8a57a47f9fab", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1222.818, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "498ec09180d3618827830f75", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1086.745, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8bca2e9bae6525115017e486", "case_id": "smart-home-android/clean/grounding/more", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1722.61, "cost": 0.0000878, "schemaValid": true, "pointHit": true, "boxIoU": 0.961403030837665, "boxIoUAt50": true }, { "attempt_id": "2f924f954b30bd2debb3e0d4", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2224.991, "cost": 0.0001187, "schemaValid": true, "pointHit": true, "boxIoU": 0.8606775633691806, "boxIoUAt50": true }, { "attempt_id": "ef55cb2ae711c48835666f7f", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1528.435, "cost": 0.0000878, "schemaValid": true, "pointHit": true, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "6ebb8ad88896130f8d8e4462", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1484.237, "cost": 0.0000877, "schemaValid": true, "pointHit": true, "boxIoU": 0.984338756857949, "boxIoUAt50": true }, { "attempt_id": "d52a62b715595a1c219b81b1", "case_id": "smart-home-android/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1817.299, "cost": 0.0001144, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f23306e3aa8106b31cddd6de", "case_id": "smart-home-android/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4629.608, "cost": 0.0002709, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "47160aa0231a961b887859e1", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1178.92, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "55961c6ae3a018907c3111cd", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1454.979, "cost": 0.0000617, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "adc6a3e313cce10f9f76f109", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2812.533, "cost": 0.00008809, "schemaValid": true, "pointHit": true, "boxIoU": 0.9675787247276455, "boxIoUAt50": true }, { "attempt_id": "38236b27557fa1d716405582", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1695.894, "cost": 0.00003759, "schemaValid": true, "pointHit": true, "boxIoU": 0.948336705308827, "boxIoUAt50": true }, { "attempt_id": "cc96e157cda05dc30a251aa2", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2451.018, "cost": 0.00006459, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "abd31cbaedb16836fa765394", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3834.119, "cost": 0.0001656, "schemaValid": true, "pointHit": true, "boxIoU": 0.4201883727538216, "boxIoUAt50": false }, { "attempt_id": "5d5c18e33e2ea313f3facab7", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3663.385, "cost": 0.00006566, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "865554f7ac318fbca68961bb", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3673.535, "cost": 0.00015616, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "45401a474ce4b7577d885847", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1854.282, "cost": 0.00002418, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "16f43b45d458fd6e55497cd7", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1325.119, "cost": 0.00002419, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ccc580aac097997af68c6aa5", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3446.492, "cost": 0.00014158, "schemaValid": true, "pointHit": true, "boxIoU": 0.9861414997178546, "boxIoUAt50": true }, { "attempt_id": "20b98bb064cc259162e916cd", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2062.056, "cost": 0.00007309, "schemaValid": true, "pointHit": true, "boxIoU": 0.9977190331510077, "boxIoUAt50": true }, { "attempt_id": "428c790b4b96083952e0e5aa", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2003.666, "cost": 0.00006658, "schemaValid": true, "pointHit": true, "boxIoU": 0.9782124823785875, "boxIoUAt50": true }, { "attempt_id": "a6619bae8f7758c8646a62ff", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2300.159, "cost": 0.00007759, "schemaValid": true, "pointHit": true, "boxIoU": 0.48601655892005474, "boxIoUAt50": false }, { "attempt_id": "add4b0eba700ed0a2cc499ba", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1754.822, "cost": 0.00005066, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "521c180238d6ae3deac581ae", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3117.207, "cost": 0.00012866, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.21204058307749196, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "49b2feb38dc26178707720e0", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1270.779, "cost": 0.00002418, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7adb8b5b7b611450cf9b4045", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1320.175, "cost": 0.00002418, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "25c981a327440c23180371c7", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2500.328, "cost": 0.00008459, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "1494f52f5b9e4b61d39ac76e", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2750.991, "cost": 0.00007709, "schemaValid": true, "pointHit": true, "boxIoU": 0.9722465288420963, "boxIoUAt50": true }, { "attempt_id": "0c3fa0beb3510cbad7ef3555", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2788.757, "cost": 0.00009809, "schemaValid": true, "pointHit": true, "boxIoU": 0.4384239198628043, "boxIoUAt50": false }, { "attempt_id": "f8f20dd753902b2f946e4e6d", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2317.009, "cost": 0.00008712, "schemaValid": true, "pointHit": true, "boxIoU": 0.9757049108661706, "boxIoUAt50": true }, { "attempt_id": "ae5e2322023dd077ebd86598", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3034.051, "cost": 0.00005216, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1ab29638a18e3e3f630fc676", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3081.462, "cost": 0.00010216, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "52ddc848c304b52feb8b6e76", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1438.798, "cost": 0.00002418, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "616b0833af564afd36b48d31", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1709.003, "cost": 0.00002418, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "28d59fcd5261021eb0796d19", "case_id": "banking-ios/clean/grounding/profile", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2409.159, "cost": 0.0001457, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774944177009569, "boxIoUAt50": true }, { "attempt_id": "19ffc8ac620a50e2b8b6ea53", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2900.766, "cost": 0.0001197, "schemaValid": true, "pointHit": true, "boxIoU": 0.8476522453114688, "boxIoUAt50": true }, { "attempt_id": "99ab4711741b3ad32ac613cb", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2287.288, "cost": 0.0001397, "schemaValid": true, "pointHit": true, "boxIoU": 0.34225904700074894, "boxIoUAt50": false }, { "attempt_id": "916930672d363bf475ec41b9", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2514.176, "cost": 0.0001141, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "06ff7608a97846c32a39862c", "case_id": "banking-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2214.097, "cost": 0.0001154, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "75c78506d3cc6a58e40bcdd9", "case_id": "banking-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 5430.274, "cost": 0.0002999, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "15fb86f2a5c3faf1df77b61f", "case_id": "banking-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1446.016, "cost": 0.0000615, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ca39e4b5fbb8afc44a3422e7", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1263.103, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9de0797b26ebb9db1bee807c", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2158.729, "cost": 0.0001245, "schemaValid": true, "pointHit": true, "boxIoU": 0.9846068132952047, "boxIoUAt50": true }, { "attempt_id": "e00107c7404bee253f4f70ea", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2229.396, "cost": 0.0001157, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "803d17049b819456b8094dec", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2657.36, "cost": 0.0001562, "schemaValid": true, "pointHit": true, "boxIoU": 0.728029984198174, "boxIoUAt50": true }, { "attempt_id": "5980af875f6e22be5341cec6", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3249.559, "cost": 0.0001647, "schemaValid": true, "pointHit": true, "boxIoU": 0.7279606031598126, "boxIoUAt50": true }, { "attempt_id": "3dc732b999e3e2a699d5cde3", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1773.751, "cost": 0.0001064, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "082998be35f44155c721e27c", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2891.918, "cost": 0.0001734, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.6250531328258307, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7d918626499533191b4c92dc", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1244.196, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c2627c59a1af08c9711991f8", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1056.756, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ae762673cec8f2903a65fc07", "case_id": "smart-home-android/clean/grounding/more", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2335.846, "cost": 0.0001453, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774944177009569, "boxIoUAt50": true }, { "attempt_id": "c0ac827bd44b6a9e9dd6daa8", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2182.697, "cost": 0.0001337, "schemaValid": true, "pointHit": true, "boxIoU": 0.8476522453114688, "boxIoUAt50": true }, { "attempt_id": "73503ce29abea04671643f89", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1719.918, "cost": 0.0000878, "schemaValid": true, "pointHit": true, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "74951b9e44ecc61cc47373b9", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1510.192, "cost": 0.0000877, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "e7a9c213ea1b34240e03be80", "case_id": "smart-home-android/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2432.019, "cost": 0.0001099, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6920f994985ddcd4102bc9be", "case_id": "smart-home-android/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 5865.312, "cost": 0.0002644, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f3e8127ef87bd39f58aca3ab", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1473.181, "cost": 0.0000616, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8c960641d31a4dd06b4b1797", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1660.684, "cost": 0.0000617, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "openai/gpt-6.1-sol": { "name": "GPT-6.1 Sol", "classification_only": false, "point_source": "independently predicted", "planned_requests": 144, "completed_requests": 144, "transport_attempts": 144, "request_failures": 0, "invalid_responses": 0, "resolved_models": [ "openai/gpt-6.1-sol" ], "grounding": { "metrics": { "pointAccuracy": 1, "meanIoU": 0.8117632627487608, "iouAt50": 0.8194444444444444, "schemaValidRate": 1 }, "hits": 72, "count": 72, "consistency": [ { "case_id": "commerce-catalog-desktop/clean/grounding/search", "hit_consistent": true, "meanPairwisePointDistance": 0.013333333333321207, "meanPairwiseBoxIoU": 0.9984425611906432 }, { "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9999999999999986 }, { "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.999999999999999 }, { "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "hit_consistent": true, "meanPairwisePointDistance": 0.03333333333333333, "meanPairwiseBoxIoU": 0.9950104208864965 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1.0000000000000007 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "hit_consistent": true, "meanPairwisePointDistance": 0.03771236166327503, "meanPairwiseBoxIoU": 0.9994994081568652 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "hit_consistent": true, "meanPairwisePointDistance": 0.014907119850018935, "meanPairwiseBoxIoU": 0.9992547484457918 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "hit_consistent": true, "meanPairwisePointDistance": 0.029814239700003978, "meanPairwiseBoxIoU": 0.39772913978587704 }, { "case_id": "iot-control-desktop/clean/grounding/devices-nav", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "iot-control-desktop/clean/grounding/restart-device", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1.0000000000000004 }, { "case_id": "iot-control-desktop/clean/grounding/auto-mode", "hit_consistent": true, "meanPairwisePointDistance": 0.03373754914370479, "meanPairwiseBoxIoU": 0.9967963756820318 }, { "case_id": "iot-control-desktop/clean/grounding/pump-card", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9908216411564625 }, { "case_id": "banking-ios/clean/grounding/profile", "hit_consistent": true, "meanPairwisePointDistance": 0.2748737083744951, "meanPairwiseBoxIoU": 0.9862819968887008 }, { "case_id": "banking-ios/clean/grounding/transaction-search", "hit_consistent": true, "meanPairwisePointDistance": 0.3333333333333333, "meanPairwiseBoxIoU": 0.9991163055378851 }, { "case_id": "banking-ios/clean/grounding/hide-balance", "hit_consistent": true, "meanPairwisePointDistance": 94, "meanPairwiseBoxIoU": 0.9626666666666667 }, { "case_id": "banking-ios/clean/grounding/transfer", "hit_consistent": true, "meanPairwisePointDistance": 0.03333333333330302, "meanPairwiseBoxIoU": 0.9995677822671308 }, { "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9897432750652854 }, { "case_id": "flight-pass-ios/clean/grounding/check-in", "hit_consistent": true, "meanPairwisePointDistance": 0.09999999999998484, "meanPairwiseBoxIoU": 1.0000000000000002 }, { "case_id": "flight-pass-ios/clean/grounding/trips-tab", "hit_consistent": true, "meanPairwisePointDistance": 1.3333333333333333, "meanPairwiseBoxIoU": 0.4675527282273501 }, { "case_id": "flight-pass-ios/clean/grounding/profile-tab", "hit_consistent": true, "meanPairwisePointDistance": 2.7617944285953464, "meanPairwiseBoxIoU": 0.9108114249910084 }, { "case_id": "smart-home-android/clean/grounding/more", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "smart-home-android/clean/grounding/room-search", "hit_consistent": true, "meanPairwisePointDistance": 0.3333333333333333, "meanPairwiseBoxIoU": 0.9991163055378851 }, { "case_id": "smart-home-android/clean/grounding/thermostat-card", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1.0000000000000007 }, { "case_id": "smart-home-android/clean/grounding/add-device", "hit_consistent": true, "meanPairwisePointDistance": 0.06666666666668182, "meanPairwiseBoxIoU": 1.0000000000000002 } ] }, "presence": { "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "correct": 36, "count": 36 }, "defects": { "metrics": { "balancedAccuracy": 0.9166666666666667, "recall": 0.8333333333333334, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.3333333333333333, "meanLocalizationIoU": 0.1805220788222434, "schemaValidRate": 1 }, "correct": 33, "count": 36, "category_correct": 6, "category_count": 18, "region_supported": true }, "operations_by_track": { "grounding": { "count": 72, "transport_attempts": 72, "p50_ms": 2810.699, "p95_ms": 3923.5220500000005, "cost_usd": 0.14474900000000007 }, "element-presence": { "count": 36, "transport_attempts": 36, "p50_ms": 2181.1975, "p95_ms": 2677.19625, "cost_usd": 0.05215229999999994 }, "layout-defect": { "count": 36, "transport_attempts": 36, "p50_ms": 3060.2264999999998, "p95_ms": 7052.254500000001, "cost_usd": 0.08343520000000001 } }, "operations": { "p50_ms": 2675.1755, "p95_ms": 4880.8886999999995, "cost_usd": 0.28033649999999993, "missing_cost_count": 0 }, "attempts": [ { "attempt_id": "1aa9cefbe7bdb26dfb83c4e4", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3727.256, "cost": 0.0050585, "schemaValid": true, "pointHit": true, "boxIoU": 0.9750766276000058, "boxIoUAt50": true }, { "attempt_id": "ae878aa2b8980657122b2ff2", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3411.657, "cost": 0.0050585, "schemaValid": true, "pointHit": true, "boxIoU": 0.9996328792934548, "boxIoUAt50": true }, { "attempt_id": "dff3ce39d04dd0a071f065e6", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2754.683, "cost": 0.0050785, "schemaValid": true, "pointHit": true, "boxIoU": 0.8295838955510985, "boxIoUAt50": true }, { "attempt_id": "a65c09021fc92198b0157c55", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2995.053, "cost": 0.005081, "schemaValid": true, "pointHit": true, "boxIoU": 0.0641852129394228, "boxIoUAt50": false }, { "attempt_id": "643d81a513341bd920dd000c", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3794.741, "cost": 0.005001, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "54c3910081129db5ae226140", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 5646.198, "cost": 0.005951, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.20731497653850814, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c0a536ef5217fa649202ec9d", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2035.271, "cost": 0.004366, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "309c8bd4d22b5f069310fe29", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2521.871, "cost": 0.0043685, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6daacab7c70cdf45846afeaf", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 4019.964, "cost": 0.005036, "schemaValid": true, "pointHit": true, "boxIoU": 0.9998488137461157, "boxIoUAt50": true }, { "attempt_id": "fda0a48a07532efa6c1a4d96", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3040.171, "cost": 0.0050785, "schemaValid": true, "pointHit": true, "boxIoU": 0.9986887870312527, "boxIoUAt50": true }, { "attempt_id": "d1cbc706d417fbb127a027ba", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3422.749, "cost": 0.005076, "schemaValid": true, "pointHit": true, "boxIoU": 0.9755949953274832, "boxIoUAt50": true }, { "attempt_id": "43324bd4daccd2e63f46b31e", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3782.482, "cost": 0.0054585, "schemaValid": true, "pointHit": true, "boxIoU": 0.04014943221496962, "boxIoUAt50": false }, { "attempt_id": "b9100d29cc193eb99159595e", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2679.778, "cost": 0.005001, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3bb85e018741372be1d823b0", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3751.618, "cost": 0.005291, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b227adb2ffbe835e2bbff72f", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2175.855, "cost": 0.004366, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5acd793375c13f04f86a1758", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2133.199, "cost": 0.004366, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2bcc3e7a9ed70400012f4b7e", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3297.065, "cost": 0.0050785, "schemaValid": true, "pointHit": true, "boxIoU": 0.9998959481009814, "boxIoUAt50": true }, { "attempt_id": "9a1d6e50226915c48b355ec4", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2721.961, "cost": 0.0050585, "schemaValid": true, "pointHit": true, "boxIoU": 0.9968252140917356, "boxIoUAt50": true }, { "attempt_id": "b234c00d069967e3f564c644", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2668.439, "cost": 0.0050785, "schemaValid": true, "pointHit": true, "boxIoU": 0.03974687097878154, "boxIoUAt50": false }, { "attempt_id": "749fb30cbc43d5e4493c9f42", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2611.363, "cost": 0.004966, "schemaValid": true, "pointHit": true, "boxIoU": 0.9636572416795217, "boxIoUAt50": true }, { "attempt_id": "53e40051135abde1c9bf0e63", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2515.584, "cost": 0.005001, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d64e204020bebb4c0dfad1a8", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2670.573, "cost": 0.005261, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "88c07a4fdaa0159e92cd3929", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2461.443, "cost": 0.004366, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a593e614da076fbbaff78e43", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2268.053, "cost": 0.004366, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3899f255f8394d5c87b59665", "case_id": "banking-ios/clean/grounding/profile", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2878.188, "cost": 0.001754, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774944177009569, "boxIoUAt50": true }, { "attempt_id": "6001132ca10d3d65f26fea87", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2620.321, "cost": 0.001854, "schemaValid": true, "pointHit": true, "boxIoU": 0.8822463488468371, "boxIoUAt50": true }, { "attempt_id": "e5b0706b20aa768d6f4ac7c2", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2781.928, "cost": 0.002034, "schemaValid": true, "pointHit": true, "boxIoU": 0.3443811619354286, "boxIoUAt50": false }, { "attempt_id": "4ffc3c4e1a41b83432378608", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3094.105, "cost": 0.001852, "schemaValid": true, "pointHit": true, "boxIoU": 0.9999222786605868, "boxIoUAt50": true }, { "attempt_id": "b72d938de7bc805eeb3d5bc8", "case_id": "banking-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2960.702, "cost": 0.001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bf8303f70d10bc63fd10bef2", "case_id": "banking-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 8637.379, "cost": 0.003608, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dd5dbbffeaaf542d8e670a56", "case_id": "banking-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1869.319, "cost": 0.00123, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "aac800ad39cd3b0435c0209c", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2351.919, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ba373ef2f1dc6e33658cb09e", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2482.225, "cost": 0.00184, "schemaValid": true, "pointHit": true, "boxIoU": 0.9999910864673834, "boxIoUAt50": true }, { "attempt_id": "4aa6dfd8e46b1343e475d4d1", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 4767.531, "cost": 0.001834, "schemaValid": true, "pointHit": true, "boxIoU": 0.9999222786605868, "boxIoUAt50": true }, { "attempt_id": "b179652c2950aa57481e44b1", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2425.443, "cost": 0.001754, "schemaValid": true, "pointHit": true, "boxIoU": 0.3310057326132378, "boxIoUAt50": false }, { "attempt_id": "a0b5d1c45f151fa364d7e26f", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2848.669, "cost": 0.001754, "schemaValid": true, "pointHit": true, "boxIoU": 0.7495253496193734, "boxIoUAt50": true }, { "attempt_id": "30fd031ea255c61cf3779bad", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2393.913, "cost": 0.001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "16c9392e5c2784e8c3dd7fb1", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3863.427, "cost": 0.002648, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.9903764919496771, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a034d4e37dd8307c15b2efd8", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1981.853, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6ec3a63d300f3a43b51d3faf", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2532.932, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "10eb010a5db8c97cab3246f5", "case_id": "smart-home-android/clean/grounding/more", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2271.928, "cost": 0.001756, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774944177009569, "boxIoUAt50": true }, { "attempt_id": "a86fa53b7e214674de4e60a5", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2444.987, "cost": 0.001854, "schemaValid": true, "pointHit": true, "boxIoU": 0.8822463488468371, "boxIoUAt50": true }, { "attempt_id": "a00cd012aeae7389f940cd16", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2775.358, "cost": 0.001856, "schemaValid": true, "pointHit": true, "boxIoU": 0.9993806609971408, "boxIoUAt50": true }, { "attempt_id": "9398c2225bd024af0b6b0e08", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2717.745, "cost": 0.001834, "schemaValid": true, "pointHit": true, "boxIoU": 0.9999222786605868, "boxIoUAt50": true }, { "attempt_id": "399289b17c9f16d75c77ce63", "case_id": "smart-home-android/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2437.725, "cost": 0.001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d1cbc3f14158adf3d41ca4c7", "case_id": "smart-home-android/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 5474.587, "cost": 0.002958, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "59a6cb33761690eaf8efb66d", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1665.268, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e747c02aa6afd85104b902ab", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2359.088, "cost": 0.001234, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ef4fe69aa882ae7ac5ceb9d6", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2801.089, "cost": 0.0006689, "schemaValid": true, "pointHit": true, "boxIoU": 0.9750766276000058, "boxIoUAt50": true }, { "attempt_id": "47143dd4102ed105765f007e", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2607.398, "cost": 0.0006689, "schemaValid": true, "pointHit": true, "boxIoU": 0.9996328792934548, "boxIoUAt50": true }, { "attempt_id": "760cf28b810076430d5d297f", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2822.568, "cost": 0.0006889, "schemaValid": true, "pointHit": true, "boxIoU": 0.8295838955510985, "boxIoUAt50": true }, { "attempt_id": "e4b4513681ab7172ac4466b2", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3147.449, "cost": 0.000689, "schemaValid": true, "pointHit": true, "boxIoU": 0.0641852129394228, "boxIoUAt50": false }, { "attempt_id": "1976d447bd25b0ef6b2afbe5", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2996.29, "cost": 0.0004746, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0e46ecc490e3def374d697de", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4900.893, "cost": 0.0013446, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.22277640118340725, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "50dd25a35cdf9fc6244ef994", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2409.111, "cost": 0.0003148, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "672bcb8417dcfcc7a28e7682", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2604.907, "cost": 0.0003149, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ec1e5eae831f85d7cb2d3751", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2800.162, "cost": 0.0006488, "schemaValid": true, "pointHit": true, "boxIoU": 0.9998488137461157, "boxIoUAt50": true }, { "attempt_id": "2235d15ad0744f1d09f2b1d3", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2484.15, "cost": 0.0006889, "schemaValid": true, "pointHit": true, "boxIoU": 0.9986887870312527, "boxIoUAt50": true }, { "attempt_id": "da09c38eed9410ef533a3d9f", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2846.967, "cost": 0.0006888, "schemaValid": true, "pointHit": true, "boxIoU": 0.9755949953274832, "boxIoUAt50": true }, { "attempt_id": "275a699e6ba6df364c00ecd8", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3917.614, "cost": 0.0012389, "schemaValid": true, "pointHit": true, "boxIoU": 0.04014943221496962, "boxIoUAt50": false }, { "attempt_id": "dcefdfdf090f38c637ea15d1", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1978.809, "cost": 0.0004746, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b6877c51212eebfe13d11217", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3466.29, "cost": 0.0007746, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c984a45a367eb07cffe8efec", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2604.878, "cost": 0.0003148, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1c29bc2c8524c8dacac12664", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2424.073, "cost": 0.0003148, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "89857f7a0667aaf8dde1a581", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3337.013, "cost": 0.0006889, "schemaValid": true, "pointHit": true, "boxIoU": 0.9998959481009814, "boxIoUAt50": true }, { "attempt_id": "d63dc66fec86f4db73014906", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2940.145, "cost": 0.0006689, "schemaValid": true, "pointHit": true, "boxIoU": 0.9968252140917356, "boxIoUAt50": true }, { "attempt_id": "a41f421d16cb09f4c989dfd3", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4135.533, "cost": 0.0006889, "schemaValid": true, "pointHit": true, "boxIoU": 0.03974687097878154, "boxIoUAt50": false }, { "attempt_id": "57ab86e4b136aa64716d8a8a", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2509.64, "cost": 0.0005692, "schemaValid": true, "pointHit": true, "boxIoU": 0.9756284428926467, "boxIoUAt50": true }, { "attempt_id": "11d974eb027611a54e545d9e", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2473.046, "cost": 0.0004746, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "baa5f44bf897fa0b1ff8899b", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2897.128, "cost": 0.0007546, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "5cc5b14af46d4106f4142f87", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1936.118, "cost": 0.0003148, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e9b4ac0eb06dc228d8bfa774", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2498.871, "cost": 0.0003148, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "563b967cb594ac8957688fbb", "case_id": "banking-ios/clean/grounding/profile", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2888.886, "cost": 0.001874, "schemaValid": true, "pointHit": true, "boxIoU": 0.9976839085242092, "boxIoUAt50": true }, { "attempt_id": "cc28222a8cd38c33c8efeec6", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2575.08, "cost": 0.001854, "schemaValid": true, "pointHit": true, "boxIoU": 0.8814547422495315, "boxIoUAt50": true }, { "attempt_id": "a99a80d8fb750f14fd9a944f", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2404.919, "cost": 0.001754, "schemaValid": true, "pointHit": true, "boxIoU": 0.34225904700074894, "boxIoUAt50": false }, { "attempt_id": "bc3f5e78bb02db33450d972f", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2814.165, "cost": 0.001852, "schemaValid": true, "pointHit": true, "boxIoU": 0.9999222786605868, "boxIoUAt50": true }, { "attempt_id": "d08c2ba618b5ea6681476971", "case_id": "banking-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1893.448, "cost": 0.001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2b7f5c807600496813fdf1cc", "case_id": "banking-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 7422.135, "cost": 0.003598, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e613969fd4ed9a6b5b232d41", "case_id": "banking-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1719.844, "cost": 0.00123, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6918adee4ea1e836674b04f9", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2024.313, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a93b87565275e90044e37273", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3135.648, "cost": 0.00176, "schemaValid": true, "pointHit": true, "boxIoU": 0.9846068132952047, "boxIoUAt50": true }, { "attempt_id": "c449331a2481d1e8ecb65d29", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2862.011, "cost": 0.001854, "schemaValid": true, "pointHit": true, "boxIoU": 0.9999222786605868, "boxIoUAt50": true }, { "attempt_id": "2ea3d5e82786e6420dd72b63", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2243.074, "cost": 0.001754, "schemaValid": true, "pointHit": true, "boxIoU": 0.8251020463876413, "boxIoUAt50": true }, { "attempt_id": "248602ed5185954cc75878bf", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2465.203, "cost": 0.001754, "schemaValid": true, "pointHit": true, "boxIoU": 0.8047134388518015, "boxIoUAt50": true }, { "attempt_id": "936fa227946822e1aaf262f1", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1990.11, "cost": 0.001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a224e0b3bc5cf08d8fd48fe3", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4434.157, "cost": 0.002898, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.9903764919496771, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c4a488cb30375f525e0fc24f", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1937.228, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "930d99433b1319fad821c6fc", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2894.064, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e9d395f17e47d5969efa2150", "case_id": "smart-home-android/clean/grounding/more", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2351.631, "cost": 0.001756, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774944177009569, "boxIoUAt50": true }, { "attempt_id": "1d6c527d636739f054839e43", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2577.413, "cost": 0.001834, "schemaValid": true, "pointHit": true, "boxIoU": 0.8822463488468371, "boxIoUAt50": true }, { "attempt_id": "034092352f4f8e84c13083de", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2818.362, "cost": 0.001856, "schemaValid": true, "pointHit": true, "boxIoU": 0.9993806609971408, "boxIoUAt50": true }, { "attempt_id": "320ddf8199a597288a041fd1", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2707.67, "cost": 0.001854, "schemaValid": true, "pointHit": true, "boxIoU": 0.9999222786605868, "boxIoUAt50": true }, { "attempt_id": "777253712e8c4c3a8e97a954", "case_id": "smart-home-android/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2128.533, "cost": 0.001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fcfbdd91f9f3f03e0c771d62", "case_id": "smart-home-android/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 5017.264, "cost": 0.002958, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b448ac802ea960512e997648", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2053.789, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "158320a808c7962fa79032ac", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1982.109, "cost": 0.001234, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "389ab3aab226e8b831b92946", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3930.743, "cost": 0.0006689, "schemaValid": true, "pointHit": true, "boxIoU": 0.9741377961555361, "boxIoUAt50": true }, { "attempt_id": "5662adb39eec699c9e6d72e1", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2400.988, "cost": 0.0006689, "schemaValid": true, "pointHit": true, "boxIoU": 0.9996328792934548, "boxIoUAt50": true }, { "attempt_id": "8593f4d0cf39f40adb0d2901", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3392.701, "cost": 0.0006889, "schemaValid": true, "pointHit": true, "boxIoU": 0.8295838955510985, "boxIoUAt50": true }, { "attempt_id": "128cf642c74553827239c0f8", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2850.149, "cost": 0.000689, "schemaValid": true, "pointHit": true, "boxIoU": 0.0642056179775243, "boxIoUAt50": false }, { "attempt_id": "eab47c7122cb13539e4f0bd3", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3402.546, "cost": 0.0005546, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d8273f574af4891f2707a792", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 4582.417, "cost": 0.0013146, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2134999243532805, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f05c715badc2908c79ce0e73", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2227.486, "cost": 0.0003148, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "83cd59965fb92db5a1492b3e", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 3348.516, "cost": 0.0003149, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f909646deb75593e5892ecc6", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2940.726, "cost": 0.0006488, "schemaValid": true, "pointHit": true, "boxIoU": 0.9998488137461157, "boxIoUAt50": true }, { "attempt_id": "41f352d9ff76d71d0894c3c5", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2858.633, "cost": 0.0006889, "schemaValid": true, "pointHit": true, "boxIoU": 0.9994390158369846, "boxIoUAt50": true }, { "attempt_id": "3481222c2f16d3276562f9d7", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2807.233, "cost": 0.0006888, "schemaValid": true, "pointHit": true, "boxIoU": 0.975330629266659, "boxIoUAt50": true }, { "attempt_id": "533acfaaf78b3245d50c5d0a", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3482.55, "cost": 0.0011389, "schemaValid": true, "pointHit": true, "boxIoU": 0.41565265842331295, "boxIoUAt50": false }, { "attempt_id": "820cf0bef0b09ccae38fb436", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2594.827, "cost": 0.0004746, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3111efb86fcd05d69863a838", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3588.094, "cost": 0.0007746, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "edbf098a91882e1b006ff77a", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2548.273, "cost": 0.0003148, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7c8dc982400bc55dd25ce6c4", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2034.64, "cost": 0.0003148, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "cd21572c36d7845d14eb0a36", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3036.438, "cost": 0.0006889, "schemaValid": true, "pointHit": true, "boxIoU": 0.9998959481009814, "boxIoUAt50": true }, { "attempt_id": "feab225c1a9f39e645c4ded3", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3256.025, "cost": 0.0006689, "schemaValid": true, "pointHit": true, "boxIoU": 0.9968252140917356, "boxIoUAt50": true }, { "attempt_id": "1b27b16a89c763e3facc5ee7", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2988.935, "cost": 0.0006889, "schemaValid": true, "pointHit": true, "boxIoU": 0.039743936390056224, "boxIoUAt50": false }, { "attempt_id": "ce70a7a5aabfbffea7ccff95", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2796.139, "cost": 0.0005692, "schemaValid": true, "pointHit": true, "boxIoU": 0.9637247179241175, "boxIoUAt50": true }, { "attempt_id": "ef909c75bd9288aef8120aea", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3124.163, "cost": 0.0004746, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f7c7f976a0d8f2a5d5594fb1", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2841.267, "cost": 0.0006846, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "918a6f4d0472e4696ce050b0", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2391.862, "cost": 0.0003148, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e51e041e1834a1a69a71abf5", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2186.54, "cost": 0.0003148, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e2a18d87fbe3c23cc8c54f2f", "case_id": "banking-ios/clean/grounding/profile", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3353.761, "cost": 0.001874, "schemaValid": true, "pointHit": true, "boxIoU": 0.9976839085242092, "boxIoUAt50": true }, { "attempt_id": "14be8ee30a74d709e101dd6c", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2371.14, "cost": 0.001834, "schemaValid": true, "pointHit": true, "boxIoU": 0.8822463488468371, "boxIoUAt50": true }, { "attempt_id": "ed147db2bd474324841d910c", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3369.664, "cost": 0.002314, "schemaValid": true, "pointHit": true, "boxIoU": 0.32515361407108945, "boxIoUAt50": false }, { "attempt_id": "e0b0f4c8d9744a57736831f2", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2540.342, "cost": 0.001852, "schemaValid": true, "pointHit": true, "boxIoU": 0.9993977147923093, "boxIoUAt50": true }, { "attempt_id": "3888de27b51ea0c965c6edfd", "case_id": "banking-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2054.937, "cost": 0.001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "85e89d74cadf0d807e871837", "case_id": "banking-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 6928.961, "cost": 0.003138, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2e30b61e0a0799f3bc084cee", "case_id": "banking-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1848.5, "cost": 0.00123, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e07ee5f41acad9a8f45c15c1", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2600.43, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "88c8caffa9716f35f8a7c8b7", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2123.487, "cost": 0.00176, "schemaValid": true, "pointHit": true, "boxIoU": 0.9846068132952047, "boxIoUAt50": true }, { "attempt_id": "8d5b5c601ee7db0f73a4e1c9", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2211.873, "cost": 0.001854, "schemaValid": true, "pointHit": true, "boxIoU": 0.9999222786605868, "boxIoUAt50": true }, { "attempt_id": "0a1eee138661eede14bce69a", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3528.991, "cost": 0.002254, "schemaValid": true, "pointHit": true, "boxIoU": 0.7324428188465387, "boxIoUAt50": true }, { "attempt_id": "774301d981ac1646b3b1aa7e", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2347.102, "cost": 0.001754, "schemaValid": true, "pointHit": true, "boxIoU": 0.793244539639715, "boxIoUAt50": true }, { "attempt_id": "aeebc2eda432cd3e9becc046", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2033.566, "cost": 0.001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d4585cffda02498afa5a89cb", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 4191.557, "cost": 0.002828, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.6250531328258307, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f15cfa5a15fa8acf52f8dd94", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1909.63, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7edf6408d5d7c905ff962025", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2124.6, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "444ceaa8a32aac5a5bf8e817", "case_id": "smart-home-android/clean/grounding/more", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2304.141, "cost": 0.001756, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774944177009569, "boxIoUAt50": true }, { "attempt_id": "4d69c9015adf95a1683288e1", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2598.149, "cost": 0.001854, "schemaValid": true, "pointHit": true, "boxIoU": 0.8814547422495315, "boxIoUAt50": true }, { "attempt_id": "004220374e016b0c6eae0154", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2752.261, "cost": 0.001856, "schemaValid": true, "pointHit": true, "boxIoU": 0.9993806609971408, "boxIoUAt50": true }, { "attempt_id": "ea31ce7ca48cf19034d45906", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2493.773, "cost": 0.001854, "schemaValid": true, "pointHit": true, "boxIoU": 0.9999222786605868, "boxIoUAt50": true }, { "attempt_id": "113a45d63fc274d31c93c26b", "case_id": "smart-home-android/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2170.281, "cost": 0.001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "03bb75b740e21b71c2f0dc2d", "case_id": "smart-home-android/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 5168.439, "cost": 0.002808, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "709efd67be90327550eae713", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2059.64, "cost": 0.001232, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0cbabee3094288b111e8d0c2", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2068.286, "cost": 0.001234, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "anthropic/claude-sonnet-5.5": { "name": "Claude Sonnet 5.5", "classification_only": false, "point_source": "independently predicted", "planned_requests": 144, "completed_requests": 144, "transport_attempts": 144, "request_failures": 0, "invalid_responses": 8, "resolved_models": [ "anthropic/claude-sonnet-5.5" ], "grounding": { "metrics": { "pointAccuracy": 0.9722222222222222, "meanIoU": 0.7476465539963829, "iouAt50": 0.7361111111111112, "schemaValidRate": 0.9722222222222222 }, "hits": 70, "count": 72, "consistency": [ { "case_id": "commerce-catalog-desktop/clean/grounding/search", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.7999999999999999 }, { "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "hit_consistent": false, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.40672782874617736 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9953703703703703 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "iot-control-desktop/clean/grounding/devices-nav", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "iot-control-desktop/clean/grounding/restart-device", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9386251899526235 }, { "case_id": "iot-control-desktop/clean/grounding/auto-mode", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "iot-control-desktop/clean/grounding/pump-card", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9762497923932901 }, { "case_id": "banking-ios/clean/grounding/profile", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.6962962962962962 }, { "case_id": "banking-ios/clean/grounding/transaction-search", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "banking-ios/clean/grounding/hide-balance", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.6857924321708116 }, { "case_id": "banking-ios/clean/grounding/transfer", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "hit_consistent": true, "meanPairwisePointDistance": 1.3333333333333333, "meanPairwiseBoxIoU": 1 }, { "case_id": "flight-pass-ios/clean/grounding/check-in", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9883040935672515 }, { "case_id": "flight-pass-ios/clean/grounding/trips-tab", "hit_consistent": false, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.8461538461538461 }, { "case_id": "flight-pass-ios/clean/grounding/profile-tab", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.8123688895940204 }, { "case_id": "smart-home-android/clean/grounding/more", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9777777777777777 }, { "case_id": "smart-home-android/clean/grounding/room-search", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9858156028368793 }, { "case_id": "smart-home-android/clean/grounding/thermostat-card", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9831223628691982 }, { "case_id": "smart-home-android/clean/grounding/add-device", "hit_consistent": true, "meanPairwisePointDistance": 1.3333333333333333, "meanPairwiseBoxIoU": 0.9333333333333332 } ] }, "presence": { "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "correct": 36, "count": 36 }, "defects": { "metrics": { "balancedAccuracy": 0.8333333333333333, "recall": 0.6666666666666666, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.5, "meanLocalizationIoU": 0.19372110680362564, "schemaValidRate": 0.8333333333333334 }, "correct": 30, "count": 36, "category_correct": 9, "category_count": 18, "region_supported": true }, "operations_by_track": { "grounding": { "count": 72, "transport_attempts": 72, "p50_ms": 2221.125, "p95_ms": 3061.4207, "cost_usd": 0.2969880000000001 }, "element-presence": { "count": 36, "transport_attempts": 36, "p50_ms": 1848.9555, "p95_ms": 2775.662, "cost_usd": 0.107838 }, "layout-defect": { "count": 36, "transport_attempts": 36, "p50_ms": 2414.8745, "p95_ms": 4669.6075, "cost_usd": 0.175446 } }, "operations": { "p50_ms": 2203.5434999999998, "p95_ms": 3988.5390500000003, "cost_usd": 0.5802720000000002, "missing_cost_count": 0 }, "attempts": [ { "attempt_id": "8cd6dd12ec9070c76a80aa08", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3958.588, "cost": 0.005402, "schemaValid": true, "pointHit": true, "boxIoU": 0.9675787247276455, "boxIoUAt50": true }, { "attempt_id": "0f5ef1070d42ff0a76236b2c", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 1, "track": "grounding", "status": "invalid_response", "wall_time_ms": 2018.401, "cost": 0.005402, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "5d6be2ea6f01becdd5f525e0", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3078.254, "cost": 0.005406, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "476f1c5ff2235e76c8256a1b", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2397.691, "cost": 0.00541, "schemaValid": true, "pointHit": true, "boxIoU": 0.3693658539807224, "boxIoUAt50": false }, { "attempt_id": "24ff446c2d6dd89eeeb4cddf", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3747.35, "cost": 0.005978, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a61613cb8462eebe2df96fe4", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 4650.768, "cost": 0.006598, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "93ff8dacee1695a1c146a5c2", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2458.759, "cost": 0.004274, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "79947c0ff8c7e60a76e0f8dd", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2240.603, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9d1ca7da1fd35cdeeaf92171", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2473.163, "cost": 0.005404, "schemaValid": true, "pointHit": true, "boxIoU": 0.8883631151322505, "boxIoUAt50": true }, { "attempt_id": "8250c1f6e4a08d73c2ba48e5", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2461.746, "cost": 0.005408, "schemaValid": true, "pointHit": true, "boxIoU": 0.9874383405985117, "boxIoUAt50": true }, { "attempt_id": "d49537be68f2a6111f46c25d", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2936.31, "cost": 0.005398, "schemaValid": true, "pointHit": true, "boxIoU": 0.9568193821923789, "boxIoUAt50": true }, { "attempt_id": "85e69fd163b14013cf787a50", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2306.417, "cost": 0.005408, "schemaValid": true, "pointHit": true, "boxIoU": 0.4822107571802293, "boxIoUAt50": false }, { "attempt_id": "189bc0f4c04d409b51e6badf", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4479.674, "cost": 0.006918, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "db80fa5c6ae5d1975b9231f8", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2545.182, "cost": 0.006178, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3acb8bc69471bb0271eab44d", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2138.496, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e3c641a7a655b1ea3d510866", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2136.992, "cost": 0.004278, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "03cd658b292a0ff7c68ac479", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3047.648, "cost": 0.005408, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "19df34bb757ba1c5c9062fe1", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2126.535, "cost": 0.005404, "schemaValid": true, "pointHit": true, "boxIoU": 0.969396579662734, "boxIoUAt50": true }, { "attempt_id": "ffbf2da11f2c45f828b0b852", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2389.581, "cost": 0.005412, "schemaValid": true, "pointHit": true, "boxIoU": 0.47834996248749234, "boxIoUAt50": false }, { "attempt_id": "417a56fb4f578a1f3e87003f", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2311.069, "cost": 0.005416, "schemaValid": true, "pointHit": true, "boxIoU": 0.9769618105852758, "boxIoUAt50": true }, { "attempt_id": "ba36f8ad67c04786d39cc080", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3978.022, "cost": 0.005938, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0830ce7b6b88c2063669697d", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4726.126, "cost": 0.006268, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07710106711068371, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3b85716b8993ed8f052ada55", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1994.851, "cost": 0.004276, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "92e725b48ffda28258efc6f0", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2348.955, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "758daf85fe1c52da00fc8925", "case_id": "banking-ios/clean/grounding/profile", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1837.072, "cost": 0.00284, "schemaValid": true, "pointHit": true, "boxIoU": 0.9028215058691854, "boxIoUAt50": true }, { "attempt_id": "771ce9f4781b104b9583aedc", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3141.563, "cost": 0.002842, "schemaValid": true, "pointHit": true, "boxIoU": 0.8385875313794983, "boxIoUAt50": true }, { "attempt_id": "4d7f9c7aabd9e4a5e106a4c5", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1921.621, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.2800231489530511, "boxIoUAt50": false }, { "attempt_id": "b513a84dc8db069eb807550b", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1819.048, "cost": 0.00284, "schemaValid": true, "pointHit": true, "boxIoU": 0.8884812064980726, "boxIoUAt50": true }, { "attempt_id": "87a9c5b5e39d29caa24f5131", "case_id": "banking-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2209.431, "cost": 0.003224, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f0b82b668b6407e1b9a3670d", "case_id": "banking-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 3401.388, "cost": 0.003784, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "58ea833af497c6dcd92a4695", "case_id": "banking-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1506.811, "cost": 0.001712, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d0e4b6a974bb2456fc3517f3", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1834.336, "cost": 0.001714, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "add9a7dc80745dee353e92d5", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2457.587, "cost": 0.002854, "schemaValid": true, "pointHit": true, "boxIoU": 0.986720876111822, "boxIoUAt50": true }, { "attempt_id": "8db1b104dbe434efc5467d3e", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1908.453, "cost": 0.002838, "schemaValid": true, "pointHit": true, "boxIoU": 0.953256795511609, "boxIoUAt50": true }, { "attempt_id": "185b4d2924fec9d1061321fe", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1887.529, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.44808931283560793, "boxIoUAt50": false }, { "attempt_id": "08c762e320f7874dc3c245d8", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2296.492, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.3895265196371395, "boxIoUAt50": false }, { "attempt_id": "e0a6e59add285c3c4906f91d", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2338.601, "cost": 0.003294, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6e9e40540b152f2c63cbafaf", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3209.677, "cost": 0.003664, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.7994914791616189, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a8f7767ff291972b6c6fd2ff", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1771.639, "cost": 0.00171, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d56ed7d7c1c1164301385bd0", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 4325.646, "cost": 0.001716, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dddccfe7ee0c6d1b7d28d386", "case_id": "smart-home-android/clean/grounding/more", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2228.634, "cost": 0.00284, "schemaValid": true, "pointHit": true, "boxIoU": 0.7099211159875921, "boxIoUAt50": true }, { "attempt_id": "1cee8e6d3f4ed5053b34f747", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 9811.382, "cost": 0.002842, "schemaValid": true, "pointHit": true, "boxIoU": 0.8385875313794983, "boxIoUAt50": true }, { "attempt_id": "a54f4d4c8868735a1d6019a3", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2250.874, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.89591680461369, "boxIoUAt50": true }, { "attempt_id": "8c65d3579fa6c00c637e3fd8", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1831.931, "cost": 0.002838, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "696b62b8e7def3b8d261a1fa", "case_id": "smart-home-android/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4382.234, "cost": 0.003444, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "49bf27d6b14439b62644f946", "case_id": "smart-home-android/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 5424.818, "cost": 0.003704, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "3aec131bbbac72764e2aa0da", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1529.91, "cost": 0.00171, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ede08150002ab9a0ed6df8ae", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1987.868, "cost": 0.001716, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b6359d5790482cacb476367e", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2133.828, "cost": 0.005402, "schemaValid": true, "pointHit": true, "boxIoU": 0.6808252623202415, "boxIoUAt50": true }, { "attempt_id": "641c3b8201b2d6b3a83ebeaa", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1888.705, "cost": 0.005402, "schemaValid": true, "pointHit": true, "boxIoU": 0.9868909507537328, "boxIoUAt50": true }, { "attempt_id": "16890eff638693964e017078", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2721.389, "cost": 0.005406, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "8c27fe67ae7c036512da71bb", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2045.431, "cost": 0.00541, "schemaValid": true, "pointHit": true, "boxIoU": 0.06907206127460082, "boxIoUAt50": false }, { "attempt_id": "d6345a48dc782acb47fa487b", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2406.625, "cost": 0.005968, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "06531eba99b102d2a4ebab2f", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3990.395, "cost": 0.006238, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.533907400604295, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "04dec8e31893e0c22df147be", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2639.256, "cost": 0.004274, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bf679e3acfb0e9b18c37139f", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2468.698, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8a389e13f5de7325a72b38b6", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2294.274, "cost": 0.005404, "schemaValid": true, "pointHit": true, "boxIoU": 0.8883631151322505, "boxIoUAt50": true }, { "attempt_id": "c5048e02eb22f2f1cf4311d8", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2146.607, "cost": 0.005408, "schemaValid": true, "pointHit": true, "boxIoU": 0.9874383405985117, "boxIoUAt50": true }, { "attempt_id": "b69492256e769e61325aa36b", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2469.594, "cost": 0.005398, "schemaValid": true, "pointHit": true, "boxIoU": 0.9568193821923789, "boxIoUAt50": true }, { "attempt_id": "1281d9f5fe4cdc29eab0b65d", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2015.692, "cost": 0.005408, "schemaValid": true, "pointHit": true, "boxIoU": 0.4822107571802293, "boxIoUAt50": false }, { "attempt_id": "24e5d9cfbfa960932796ac42", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3548.624, "cost": 0.006908, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8777700fa08e4b16c8d7bed2", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2219.919, "cost": 0.006178, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "aa44b54523e35a5827a6edd2", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1770.431, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1af2a570d8d489acf5c076af", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1684.127, "cost": 0.004278, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "469d8770f7a78e7528d17deb", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2638.223, "cost": 0.005408, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "cc429cb74479d706c89b5e24", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2200.442, "cost": 0.005404, "schemaValid": true, "pointHit": true, "boxIoU": 0.9607792211895286, "boxIoUAt50": true }, { "attempt_id": "9c03e985b19d4b85dc4638ee", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2460.947, "cost": 0.005412, "schemaValid": true, "pointHit": true, "boxIoU": 0.47834996248749234, "boxIoUAt50": false }, { "attempt_id": "c408f15855bd37b794e36204", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2206.881, "cost": 0.005416, "schemaValid": true, "pointHit": true, "boxIoU": 0.9637247179241175, "boxIoUAt50": true }, { "attempt_id": "f320f9e25238863d633aab04", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2094.82, "cost": 0.005968, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9b1800185c2be0fb845ab32f", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3483.815, "cost": 0.006258, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07710106711068371, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0842555c280714fea7013fce", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2354.398, "cost": 0.004276, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dfb281163d3e00b1e4041940", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1889.971, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6c43f4caf787f38aebcf4f32", "case_id": "banking-ios/clean/grounding/profile", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2229.683, "cost": 0.00284, "schemaValid": true, "pointHit": true, "boxIoU": 0.5048965698034543, "boxIoUAt50": true }, { "attempt_id": "177098a89f49dcc1d7c56119", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1605.425, "cost": 0.002842, "schemaValid": true, "pointHit": true, "boxIoU": 0.8385875313794983, "boxIoUAt50": true }, { "attempt_id": "7eae604ed3db916aa930b423", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1871.3, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.3694976290046174, "boxIoUAt50": false }, { "attempt_id": "cb0024c1006c2c0dae4063c3", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1825.446, "cost": 0.00284, "schemaValid": true, "pointHit": true, "boxIoU": 0.8884812064980726, "boxIoUAt50": true }, { "attempt_id": "6369f16bff2ce457b2f179ab", "case_id": "banking-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2290.041, "cost": 0.003264, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "71fb9bd7bcd8018402b5ed96", "case_id": "banking-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 2885.622, "cost": 0.003824, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "785b16af195030088e288f63", "case_id": "banking-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1606.987, "cost": 0.001712, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a0c3b9f767f30a2cdaf171c5", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1628.883, "cost": 0.001714, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e8d821d2cf5ef0aa7bf02a75", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2206.22, "cost": 0.002854, "schemaValid": true, "pointHit": true, "boxIoU": 0.986720876111822, "boxIoUAt50": true }, { "attempt_id": "edef9cd95dfda1b2de0e67e2", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2253.666, "cost": 0.002838, "schemaValid": true, "pointHit": true, "boxIoU": 0.9698963319055256, "boxIoUAt50": true }, { "attempt_id": "b0e4f6e15ca4c441403d3b60", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1868.461, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.44808931283560793, "boxIoUAt50": false }, { "attempt_id": "057bb05064ee0b404fb5abf1", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2241.744, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.4078492663832627, "boxIoUAt50": false }, { "attempt_id": "2e7f8b5e1d4590e0c7bb349d", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2200.867, "cost": 0.003324, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "373934aede85c2963617b538", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2078.275, "cost": 0.003664, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.7799916869869453, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b66e3c5a5e4ba5cf935ac2bc", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1800.963, "cost": 0.00171, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8f9407b17ab9e49c1ddaf88a", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1712.187, "cost": 0.001716, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c0650e1b9c28f0abc1c67cf7", "case_id": "smart-home-android/clean/grounding/more", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1913.523, "cost": 0.00284, "schemaValid": true, "pointHit": true, "boxIoU": 0.6943958299513459, "boxIoUAt50": true }, { "attempt_id": "8087be36b6def0a152cec9f6", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1749.617, "cost": 0.002842, "schemaValid": true, "pointHit": true, "boxIoU": 0.8385875313794983, "boxIoUAt50": true }, { "attempt_id": "2fc7afc50166289fd7d42215", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2580.717, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.89591680461369, "boxIoUAt50": true }, { "attempt_id": "35f46f23d3a2a12c1253ffef", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1762.474, "cost": 0.002838, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "d43c298cfb12bc0bc3dc007e", "case_id": "smart-home-android/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1773.534, "cost": 0.003344, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "64ef263d7489baf7a8c965c2", "case_id": "smart-home-android/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2133.53, "cost": 0.003714, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.13141091225511056, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f1a52fa70281bef08a7c576a", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2739.761, "cost": 0.00171, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "02def3cf11c541308bae64ab", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1582.704, "cost": 0.001716, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a0b876920fece369b14ad7a4", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2576.421, "cost": 0.005402, "schemaValid": true, "pointHit": true, "boxIoU": 0.6808252623202415, "boxIoUAt50": true }, { "attempt_id": "f8eca8429c0b648abfe103fe", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1920.178, "cost": 0.005402, "schemaValid": true, "pointHit": true, "boxIoU": 0.9868909507537328, "boxIoUAt50": true }, { "attempt_id": "24491506498b2fdcaca01aa0", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2998.903, "cost": 0.005406, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "bb172d40e958b4e6104a80b8", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2016.606, "cost": 0.00541, "schemaValid": true, "pointHit": true, "boxIoU": 0.06907206127460082, "boxIoUAt50": false }, { "attempt_id": "37a84b89157b8afd5cdb2be2", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2563.2, "cost": 0.005978, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "199b5f0b1e90a155b0a67960", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 3318.676, "cost": 0.006498, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "30c8cb9c3c91abb32d84b45b", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1676.678, "cost": 0.004274, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e0cac6f1f4bcae3b069a909d", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1861.058, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "99a17e29b9447325b87ac0bc", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2648.26, "cost": 0.005404, "schemaValid": true, "pointHit": true, "boxIoU": 0.8883631151322505, "boxIoUAt50": true }, { "attempt_id": "6bdaba92d27a34ee4685c5b4", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1853.927, "cost": 0.005408, "schemaValid": true, "pointHit": true, "boxIoU": 0.9912790953161096, "boxIoUAt50": true }, { "attempt_id": "aacb5effe1fc861f94dc08b7", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2329.576, "cost": 0.005398, "schemaValid": true, "pointHit": true, "boxIoU": 0.9568193821923789, "boxIoUAt50": true }, { "attempt_id": "e3b64be732950dbde2ecfe88", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2311.311, "cost": 0.005408, "schemaValid": true, "pointHit": true, "boxIoU": 0.4822107571802293, "boxIoUAt50": false }, { "attempt_id": "0be8d901e7463b1b62a4d696", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2162.55, "cost": 0.005888, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5124183a2a5e51a698143d46", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2143.817, "cost": 0.006178, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a8405d558b1a0a66267dfb1d", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2053.103, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3a373a8f1fc559b54c5f1379", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2206.744, "cost": 0.004278, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0a3961fdf87de52386bc4d5a", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2431.608, "cost": 0.005408, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "c0a896719ab779fd304d8066", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1997.527, "cost": 0.005404, "schemaValid": true, "pointHit": true, "boxIoU": 0.8969895934121618, "boxIoUAt50": true }, { "attempt_id": "77c23321de44ce99b409ebee", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2251.967, "cost": 0.005412, "schemaValid": true, "pointHit": true, "boxIoU": 0.47834996248749234, "boxIoUAt50": false }, { "attempt_id": "27b8af59a25b5802c0ca06b7", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1909.997, "cost": 0.005416, "schemaValid": true, "pointHit": true, "boxIoU": 0.9637247179241175, "boxIoUAt50": true }, { "attempt_id": "c00500b77dc49fcaaf9d994e", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2078.948, "cost": 0.005968, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "67a7a4961bbb2205c66ae77c", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2498.757, "cost": 0.006248, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07710106711068371, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e611471558475193fa33c532", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2883.365, "cost": 0.004276, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2a3bf2558a59f4c9872dcda8", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1826.756, "cost": 0.00428, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d48b21852ba8c35964540107", "case_id": "banking-ios/clean/grounding/profile", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1749.027, "cost": 0.00284, "schemaValid": true, "pointHit": true, "boxIoU": 0.9028215058691854, "boxIoUAt50": true }, { "attempt_id": "b520c2fc1aa0c30ea3cb41c4", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2711.078, "cost": 0.002842, "schemaValid": true, "pointHit": true, "boxIoU": 0.8385875313794983, "boxIoUAt50": true }, { "attempt_id": "6a1010b1856e5e54e1b3ed5e", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1764.835, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.2573744496135632, "boxIoUAt50": false }, { "attempt_id": "85d8cf93444bd9c34cbda21f", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1716.102, "cost": 0.00284, "schemaValid": true, "pointHit": true, "boxIoU": 0.8884812064980726, "boxIoUAt50": true }, { "attempt_id": "1b5e5ceb808c0108f4231809", "case_id": "banking-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1813.142, "cost": 0.003334, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "94edb3f08586571d476e803c", "case_id": "banking-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1910.036, "cost": 0.003684, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.21138376296362188, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9e85001dbe9a3ffc67d088dc", "case_id": "banking-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1704.629, "cost": 0.001712, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d783d245877bd881e847b57b", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1918.535, "cost": 0.001714, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4c32e1eaeae3022c4ad65790", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2360.963, "cost": 0.002854, "schemaValid": true, "pointHit": true, "boxIoU": 0.986720876111822, "boxIoUAt50": true }, { "attempt_id": "b6038a71c002a4f2177a52ed", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2245.281, "cost": 0.002838, "schemaValid": true, "pointHit": true, "boxIoU": 0.953256795511609, "boxIoUAt50": true }, { "attempt_id": "49d5c975cf9c3fe0bf9cabfb", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 3, "track": "grounding", "status": "invalid_response", "wall_time_ms": 2213.616, "cost": 0.002846, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "dbd62dc0f0bf90fe64344804", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1728.058, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.3420544777507347, "boxIoUAt50": false }, { "attempt_id": "716784e82d889632c921c1b4", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2116.863, "cost": 0.003324, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6457e1df270be1101223cba7", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2076.062, "cost": 0.003674, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.7994914791616189, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e9040670d9e4e28aff279172", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1836.853, "cost": 0.00171, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bbcc237727ac20a3d8e4b840", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1538.916, "cost": 0.001716, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6ce29a0fca63fa9642ff6468", "case_id": "smart-home-android/clean/grounding/more", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1577.047, "cost": 0.00284, "schemaValid": true, "pointHit": true, "boxIoU": 0.7211724012609875, "boxIoUAt50": true }, { "attempt_id": "d2fcfc7eaf2829abeac07378", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1859.773, "cost": 0.002842, "schemaValid": true, "pointHit": true, "boxIoU": 0.8227077379929957, "boxIoUAt50": true }, { "attempt_id": "10b86d7cca4a62055d39c634", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2242.607, "cost": 0.002846, "schemaValid": true, "pointHit": true, "boxIoU": 0.89591680461369, "boxIoUAt50": true }, { "attempt_id": "7cff3ac3e0fc2b137e5a9c09", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1747.015, "cost": 0.002838, "schemaValid": true, "pointHit": true, "boxIoU": 0.9203229100259851, "boxIoUAt50": true }, { "attempt_id": "f7a0ce9ff210cf3957ccde29", "case_id": "smart-home-android/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1974.195, "cost": 0.003324, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "677767f52d63e3f7f52f2f04", "case_id": "smart-home-android/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 2423.124, "cost": 0.003704, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "4563731b2a2f20f58f36be2c", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1666.775, "cost": 0.00171, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4261884a7ae43368c373f8cc", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1533.081, "cost": 0.001716, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "google/gemini-3.8-flash": { "name": "Gemini 3.8 Flash", "classification_only": false, "point_source": "independently predicted", "planned_requests": 144, "completed_requests": 144, "transport_attempts": 144, "request_failures": 0, "invalid_responses": 1, "resolved_models": [ "google/gemini-3.8-flash" ], "grounding": { "metrics": { "pointAccuracy": 1, "meanIoU": 0.733158786635567, "iouAt50": 0.7222222222222222, "schemaValidRate": 1 }, "hits": 72, "count": 72, "consistency": [ { "case_id": "commerce-catalog-desktop/clean/grounding/search", "hit_consistent": true, "meanPairwisePointDistance": 1.1380711874576983, "meanPairwiseBoxIoU": 0.9121482993772214 }, { "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9731972789115645 }, { "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9855072463768115 }, { "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "hit_consistent": true, "meanPairwisePointDistance": 3.047623487818248, "meanPairwiseBoxIoU": 0.3735022830417567 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9907245532245533 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9848484848484849 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "hit_consistent": true, "meanPairwisePointDistance": 36.807824009312064, "meanPairwiseBoxIoU": 0.383589349718382 }, { "case_id": "iot-control-desktop/clean/grounding/devices-nav", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "iot-control-desktop/clean/grounding/restart-device", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "iot-control-desktop/clean/grounding/auto-mode", "hit_consistent": true, "meanPairwisePointDistance": 1.4907119849998598, "meanPairwiseBoxIoU": 0.8358974358974359 }, { "case_id": "iot-control-desktop/clean/grounding/pump-card", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 1 }, { "case_id": "banking-ios/clean/grounding/profile", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.973814295257181 }, { "case_id": "banking-ios/clean/grounding/transaction-search", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9743589743589745 }, { "case_id": "banking-ios/clean/grounding/hide-balance", "hit_consistent": true, "meanPairwisePointDistance": 94.94750364445336, "meanPairwiseBoxIoU": 0.3305382461809871 }, { "case_id": "banking-ios/clean/grounding/transfer", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9887005649717514 }, { "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "flight-pass-ios/clean/grounding/check-in", "hit_consistent": true, "meanPairwisePointDistance": 1.3333333333333333, "meanPairwiseBoxIoU": 0.9662867718683031 }, { "case_id": "flight-pass-ios/clean/grounding/trips-tab", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.7139088857529042 }, { "case_id": "flight-pass-ios/clean/grounding/profile-tab", "hit_consistent": true, "meanPairwisePointDistance": 1.4907119849998598, "meanPairwiseBoxIoU": 0.6625914605909049 }, { "case_id": "smart-home-android/clean/grounding/more", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9799639350831496 }, { "case_id": "smart-home-android/clean/grounding/room-search", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "smart-home-android/clean/grounding/thermostat-card", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9992649761117236 }, { "case_id": "smart-home-android/clean/grounding/add-device", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9774011299435029 } ] }, "presence": { "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "correct": 36, "count": 36 }, "defects": { "metrics": { "balancedAccuracy": 0.7777777777777778, "recall": 1, "specificity": 0.5555555555555556, "falsePositiveRate": 0.4444444444444444, "categoryAccuracy": 0.7222222222222222, "meanLocalizationIoU": 0.17249622973772577, "schemaValidRate": 0.9722222222222222 }, "correct": 28, "count": 36, "category_correct": 13, "category_count": 18, "region_supported": true }, "operations_by_track": { "grounding": { "count": 72, "transport_attempts": 72, "p50_ms": 2251.326, "p95_ms": 6883.21530000001, "cost_usd": 0.10070475 }, "element-presence": { "count": 36, "transport_attempts": 36, "p50_ms": 2446.8325, "p95_ms": 2978.498, "cost_usd": 0.04322849999999999 }, "layout-defect": { "count": 36, "transport_attempts": 36, "p50_ms": 6169.6255, "p95_ms": 13290.19975, "cost_usd": 0.12155549999999998 } }, "operations": { "p50_ms": 2424.8725, "p95_ms": 10957.5103, "cost_usd": 0.2654887499999999, "missing_cost_count": 0 }, "attempts": [ { "attempt_id": "e291701d7ff011ae2451cf89", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2277.857, "cost": 0.0011835, "schemaValid": true, "pointHit": true, "boxIoU": 0.9461760832894336, "boxIoUAt50": true }, { "attempt_id": "07ae6a5cd37b61287bbc07c0", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3515.701, "cost": 0.001851, "schemaValid": true, "pointHit": true, "boxIoU": 0.9694279745720471, "boxIoUAt50": true }, { "attempt_id": "0f5af9ab8dba9fc3f42a2a75", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2568.317, "cost": 0.00119475, "schemaValid": true, "pointHit": true, "boxIoU": 0.8497662752173902, "boxIoUAt50": true }, { "attempt_id": "f7e22ada50ae4c08f13afe71", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2305.101, "cost": 0.00119175, "schemaValid": true, "pointHit": true, "boxIoU": 0.07108666306177668, "boxIoUAt50": false }, { "attempt_id": "e151853e990feb1297d488a0", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3648.516, "cost": 0.0019245, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "96015637f7da91b834ae8895", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4673.28, "cost": 0.0022845, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.205842281475725, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "188a172a8f102aee1aa6b47e", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2456.262, "cost": 0.001119, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0fd51a7ed5b51f4fd5f8bfe5", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 3092.489, "cost": 0.00147975, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e7843f3a0ed3e7c7a3cf4beb", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2076.978, "cost": 0.00109275, "schemaValid": true, "pointHit": true, "boxIoU": 0.7878590903468226, "boxIoUAt50": true }, { "attempt_id": "0f70fe121684abad04eebe14", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2167.934, "cost": 0.0011985, "schemaValid": true, "pointHit": true, "boxIoU": 0.9867547837191748, "boxIoUAt50": true }, { "attempt_id": "097cc216a24fb20734ecd2d6", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2407.716, "cost": 0.001194, "schemaValid": true, "pointHit": true, "boxIoU": 0.9528573818003828, "boxIoUAt50": true }, { "attempt_id": "ee9cda90104bc9943b1d337a", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2246.977, "cost": 0.0010485, "schemaValid": true, "pointHit": true, "boxIoU": 0.045352172222947455, "boxIoUAt50": false }, { "attempt_id": "c1ec46939cc28a09e5bf7c46", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 6434.52, "cost": 0.002907, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "22bf20233ff81f61711618b5", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4581.618, "cost": 0.0024045, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d650376fc95f02f6d0dac7dd", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2437.403, "cost": 0.001329, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7ab2322e271b7b7a16d8fe30", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2842.4, "cost": 0.00143775, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "622bef6fcb2decb0a3086881", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2458.776, "cost": 0.00118725, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "2f2560b9e8ecf82dda11fe4e", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2574, "cost": 0.00119475, "schemaValid": true, "pointHit": true, "boxIoU": 0.983106476315828, "boxIoUAt50": true }, { "attempt_id": "9b0f182bff59c34f87ecea4b", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2075.463, "cost": 0.00109725, "schemaValid": true, "pointHit": true, "boxIoU": 0.03724821107777484, "boxIoUAt50": false }, { "attempt_id": "c5261865305926d4c9e4de38", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2256.839, "cost": 0.001197, "schemaValid": true, "pointHit": true, "boxIoU": 0.9757049108661706, "boxIoUAt50": true }, { "attempt_id": "3f9df36405f53ab59413eaad", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 12601.867, "cost": 0.00653325, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5a659bf9f06ed9dcce023a3c", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4284.174, "cost": 0.00190575, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07710106711068371, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8c6d89c19ac739f2b916c83a", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2406.904, "cost": 0.0010815, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "040eebd3bb99a4903e98c703", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2220.881, "cost": 0.00125775, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0ff613a00850abc8985bd308", "case_id": "banking-ios/clean/grounding/profile", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 10159.83, "cost": 0.0045705, "schemaValid": true, "pointHit": true, "boxIoU": 0.9632107131768125, "boxIoUAt50": true }, { "attempt_id": "03d8cb39752f38fec1c0bd5a", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2528.293, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.8314631732710298, "boxIoUAt50": true }, { "attempt_id": "64efd4be9019116e70a3f795", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1824.119, "cost": 0.00103425, "schemaValid": true, "pointHit": true, "boxIoU": 0.03821574417724144, "boxIoUAt50": false }, { "attempt_id": "568692966becdeb496bf9d74", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2233.582, "cost": 0.00117975, "schemaValid": true, "pointHit": true, "boxIoU": 0.9636055492720064, "boxIoUAt50": true }, { "attempt_id": "b6c3ecf36ca3bd6e0b29dafa", "case_id": "banking-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 13110.225, "cost": 0.00671775, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "bb06b5d9f4a64ca35c920049", "case_id": "banking-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3900.082, "cost": 0.0018765, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.24878797791152998, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ad8dc88a6c31f089e19862a1", "case_id": "banking-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1895.824, "cost": 0.00106275, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fbb74a2ed7b493ce326ecfc1", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2576.792, "cost": 0.00137475, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "81844a14c4839fc4f1873180", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2340.67, "cost": 0.00117975, "schemaValid": true, "pointHit": true, "boxIoU": 0.9855955978183417, "boxIoUAt50": true }, { "attempt_id": "e27d1168c530d5da928c7bec", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2160.045, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "aa5f652a0cd8444f419e99b3", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1654.077, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.23753913364924684, "boxIoUAt50": false }, { "attempt_id": "ef1c7f9cd0d4ae55a27ea233", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1706.432, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.18602000824332987, "boxIoUAt50": false }, { "attempt_id": "d88e7c8d2e516f83e645b072", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 12773.546, "cost": 0.0061215, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "66c706e017fa02666c22129a", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4155.211, "cost": 0.00197025, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "019d260304028a0c0fb1e27e", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1988.344, "cost": 0.001056, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2ab774a086877dc7f76570d5", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2891.58, "cost": 0.00133725, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a8df714eb9d384f5dc458e76", "case_id": "smart-home-android/clean/grounding/more", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2045.633, "cost": 0.0011775, "schemaValid": true, "pointHit": true, "boxIoU": 0.9682190384633694, "boxIoUAt50": true }, { "attempt_id": "73cf39b97dae74974b966a95", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2356.644, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.8644842570776636, "boxIoUAt50": true }, { "attempt_id": "40dca7556797f5bfb797ca09", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2222.678, "cost": 0.00118125, "schemaValid": true, "pointHit": true, "boxIoU": 0.9780630943280407, "boxIoUAt50": true }, { "attempt_id": "2d133aeef167fbe889d4b504", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1549.113, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "278d030a590eb784fa617838", "case_id": "smart-home-android/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 8971.709, "cost": 0.00458775, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7981d8950f65c6ef205222fb", "case_id": "smart-home-android/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 7269.968, "cost": 0.0037365, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.16555472859395934, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e973f71eabad572ff178c9a7", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2352.87, "cost": 0.00111225, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6623f9b7edb6aea11d4a6012", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2059.88, "cost": 0.00131175, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a9427c4e4bb41880af45be38", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2532.253, "cost": 0.00108975, "schemaValid": true, "pointHit": true, "boxIoU": 0.8865178115822887, "boxIoUAt50": true }, { "attempt_id": "bec6adacad224b6ccf61d417", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2378.845, "cost": 0.00119475, "schemaValid": true, "pointHit": true, "boxIoU": 0.9895046844489798, "boxIoUAt50": true }, { "attempt_id": "74ae3fc0eba3f873045ae1c2", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2619.18, "cost": 0.00119475, "schemaValid": true, "pointHit": true, "boxIoU": 0.8497662752173902, "boxIoUAt50": true }, { "attempt_id": "a00459e50606ab02028b4145", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2499.212, "cost": 0.00119175, "schemaValid": true, "pointHit": true, "boxIoU": 0.07108666306177668, "boxIoUAt50": false }, { "attempt_id": "2dd5968f0f944cd882a0728b", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4107.558, "cost": 0.00201075, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c80c380407eebe7d42fd3095", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 10761.009, "cost": 0.0048195, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.22385626537912107, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "06e003a6741b73fc74801923", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2120.837, "cost": 0.00111525, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b34b1c90c1223071f5c5e9a8", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 3350.835, "cost": 0.001491, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "92d0ca570b64d7e7fa7a3ada", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 8746.117, "cost": 0.003969, "schemaValid": true, "pointHit": true, "boxIoU": 0.09776838415060334, "boxIoUAt50": false }, { "attempt_id": "73e53f86cf8e6efd8c643955", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2038.604, "cost": 0.0011985, "schemaValid": true, "pointHit": true, "boxIoU": 0.9798544006162435, "boxIoUAt50": true }, { "attempt_id": "3d4931294656445b5c939413", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1871.654, "cost": 0.00110025, "schemaValid": true, "pointHit": true, "boxIoU": 0.9751320104456906, "boxIoUAt50": true }, { "attempt_id": "ccf78f6ec8fa6eea0df2a6c2", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2238.05, "cost": 0.001101, "schemaValid": true, "pointHit": true, "boxIoU": 0.44196632516895223, "boxIoUAt50": false }, { "attempt_id": "c22c5076e42b3f20160b23ce", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 5954.485, "cost": 0.00291075, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a6836946322420795ef1c6f5", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3102.377, "cost": 0.001602, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "adbd3702e32c5284ff815909", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2674.361, "cost": 0.001329, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "50717878e96c9e88222e2aa6", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2940.501, "cost": 0.00163275, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "166f7a27fced07c822eee290", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2151.931, "cost": 0.00118725, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "32c4b33f001f6902c5d0765b", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2224.807, "cost": 0.00119475, "schemaValid": true, "pointHit": true, "boxIoU": 0.983106476315828, "boxIoUAt50": true }, { "attempt_id": "868a0a83a3321ae6f9dfcade", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2249.598, "cost": 0.00109725, "schemaValid": true, "pointHit": true, "boxIoU": 0.03724821107777484, "boxIoUAt50": false }, { "attempt_id": "d9a83ec999fb3bb29074c1c3", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2360.07, "cost": 0.001197, "schemaValid": true, "pointHit": true, "boxIoU": 0.9757049108661706, "boxIoUAt50": true }, { "attempt_id": "4a7d3cb052ca4101ef3b731d", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4699.444, "cost": 0.0024945, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "631352eb0cd616a61061b04e", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2142.569, "cost": 0.00115575, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07710106711068371, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c54d5ac83857c301575a191c", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2256.129, "cost": 0.001104, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4f9051563e75db4cbbe27959", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2550.79, "cost": 0.00142275, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "68ee9d6c9d33d7f6b7c315d0", "case_id": "banking-ios/clean/grounding/profile", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 5359.023, "cost": 0.00301425, "schemaValid": true, "pointHit": true, "boxIoU": 0.9632107131768125, "boxIoUAt50": true }, { "attempt_id": "e5e343256c9abce2575d43e3", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2494.426, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.8314631732710298, "boxIoUAt50": true }, { "attempt_id": "af463280223414acca45b56e", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2095.045, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.2914481267739344, "boxIoUAt50": false }, { "attempt_id": "a845a0068cba000d6c8d3ed6", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2066.723, "cost": 0.001086, "schemaValid": true, "pointHit": true, "boxIoU": 0.9636055492720064, "boxIoUAt50": true }, { "attempt_id": "967d6112cfcebe1baca8ee5a", "case_id": "banking-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 7069.501, "cost": 0.00325275, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "e88ecdb9462a5e326d91ad47", "case_id": "banking-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4051.346, "cost": 0.00201525, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.25279787019885774, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d82f46699ffe58941bb61087", "case_id": "banking-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2538.18, "cost": 0.00107025, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "acdcc734f50ae5e8331fbd89", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2066.478, "cost": 0.000876, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2e1c15f2938d99871e3e06a0", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2057.499, "cost": 0.0011835, "schemaValid": true, "pointHit": true, "boxIoU": 0.9855955978183417, "boxIoUAt50": true }, { "attempt_id": "6eb4b91c7f31d39383f34820", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1785.986, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.9704013348253773, "boxIoUAt50": true }, { "attempt_id": "5456784b78494eecb396f403", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2848.555, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.1412988898111845, "boxIoUAt50": false }, { "attempt_id": "7e42d5e8a70235a52256ada3", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2035.993, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.18602000824332987, "boxIoUAt50": false }, { "attempt_id": "b9409fc49504876d40839dc3", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 7451.347, "cost": 0.00332025, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "6068421dd60a7b37178d4e7b", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2349.126, "cost": 0.00112275, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3268d73eb37bbc49b80dcf89", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1728.55, "cost": 0.001071, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6280a7b71569981825afd0fd", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2850.666, "cost": 0.00135225, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dccd9aa9bacb1e2f951561fa", "case_id": "smart-home-android/clean/grounding/more", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2117.109, "cost": 0.0011775, "schemaValid": true, "pointHit": true, "boxIoU": 0.983183943555824, "boxIoUAt50": true }, { "attempt_id": "0e5d5029624b2d5568835362", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2175.727, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.8644842570776636, "boxIoUAt50": true }, { "attempt_id": "131f21cfa1acd828508e46e8", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2065.567, "cost": 0.00118125, "schemaValid": true, "pointHit": true, "boxIoU": 0.9780630943280407, "boxIoUAt50": true }, { "attempt_id": "845dd7f5a019cb9ee718775f", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2021.045, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.9636055492720064, "boxIoUAt50": true }, { "attempt_id": "f00f2a6862a5526ebdf42f49", "case_id": "smart-home-android/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 6384.766, "cost": 0.00300525, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "762ca2fb3e986dbaa37692cd", "case_id": "smart-home-android/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 8707.479, "cost": 0.004239, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.17068486221916163, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "64ea2d62e5b7b8bc7020dead", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2212.689, "cost": 0.001071, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "eaab77cb60e414d387d7d7b2", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2377.237, "cost": 0.00117675, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0b2c97fbc9e8040a48295a90", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2317.089, "cost": 0.00108975, "schemaValid": true, "pointHit": true, "boxIoU": 0.927996851884932, "boxIoUAt50": true }, { "attempt_id": "1d652764dbbc8c8cb6a6a2b9", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2396.366, "cost": 0.00119475, "schemaValid": true, "pointHit": true, "boxIoU": 0.9697145907600002, "boxIoUAt50": true }, { "attempt_id": "fcee54f99018aeb3e375df71", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2618.419, "cost": 0.00119475, "schemaValid": true, "pointHit": true, "boxIoU": 0.848576272828474, "boxIoUAt50": true }, { "attempt_id": "6b09ec4c796b82d1387857fe", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3036.118, "cost": 0.00119175, "schemaValid": true, "pointHit": true, "boxIoU": 0.07108666306177668, "boxIoUAt50": false }, { "attempt_id": "2e8d3b473d308c28ed2e9b4c", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 10992.187, "cost": 0.004542, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8542bc4ac77aad99eb959866", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2323.953, "cost": 0.00117825, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.20590071454340703, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e4d6e50b4f67abd37e378e6d", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2513.997, "cost": 0.00107025, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f113b3988f99d60fa64ea1e2", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2854.115, "cost": 0.0012885, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "396c840dc4250320c9c6cf56", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 9881.187, "cost": 0.00479775, "schemaValid": true, "pointHit": true, "boxIoU": 0.1143337914319348, "boxIoUAt50": false }, { "attempt_id": "f78794ba3a1990bb37511358", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2258.707, "cost": 0.00110475, "schemaValid": true, "pointHit": true, "boxIoU": 0.9906016731066115, "boxIoUAt50": true }, { "attempt_id": "750e75c6dbeeea7833ebc520", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2057.024, "cost": 0.001194, "schemaValid": true, "pointHit": true, "boxIoU": 0.9528573818003828, "boxIoUAt50": true }, { "attempt_id": "99b419d89e067ac3f0fde5d3", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2072.685, "cost": 0.001101, "schemaValid": true, "pointHit": true, "boxIoU": 0.463994758767261, "boxIoUAt50": false }, { "attempt_id": "7c32dbe8ce56a2cb5e693098", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 5593.493, "cost": 0.00252825, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5f12947b5d5c9821d5caa4b0", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 5535.615, "cost": 0.0026745, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d7e712a7af8822c9fbf651a4", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1800.781, "cost": 0.0008865, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ea7de52054a3a29292a4b5c3", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2892.83, "cost": 0.001389, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3e0222053425e5a92fae271a", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2305.593, "cost": 0.00118725, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "5aa7dd718271b2089d360154", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2360.038, "cost": 0.00119475, "schemaValid": true, "pointHit": true, "boxIoU": 0.983106476315828, "boxIoUAt50": true }, { "attempt_id": "eaea61870ada1e500dcd6af3", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2293.379, "cost": 0.00109725, "schemaValid": true, "pointHit": true, "boxIoU": 0.04633748746517444, "boxIoUAt50": false }, { "attempt_id": "4695c397e324fa0be3808d7b", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2735.997, "cost": 0.001197, "schemaValid": true, "pointHit": true, "boxIoU": 0.9757049108661706, "boxIoUAt50": true }, { "attempt_id": "3deaf411a846d2830b5b7116", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 11740.782, "cost": 0.0056595, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "59de9257d1d972c151fa0f1d", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 13830.124, "cost": 0.00605325, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07947340763716629, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "451c406ed9e320b36226eee4", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2368.272, "cost": 0.001074, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "afe6c1061d948b6517e14a11", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2584.838, "cost": 0.00137025, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a97a601fd70b8c1ac026c02c", "case_id": "banking-ios/clean/grounding/profile", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 12221.034, "cost": 0.00584175, "schemaValid": true, "pointHit": true, "boxIoU": 0.9485454523822487, "boxIoUAt50": true }, { "attempt_id": "4b627d25a372db7b6374316d", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3266.328, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.8644842570776636, "boxIoUAt50": true }, { "attempt_id": "929cd81588e3ed9fe9dea667", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2089.838, "cost": 0.00108675, "schemaValid": true, "pointHit": true, "boxIoU": 0.3788017579813557, "boxIoUAt50": false }, { "attempt_id": "603662979920381473f4b295", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1988.06, "cost": 0.00103725, "schemaValid": true, "pointHit": true, "boxIoU": 0.9802194380525583, "boxIoUAt50": true }, { "attempt_id": "840f14b9e841d18abde26836", "case_id": "banking-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 8336.67, "cost": 0.0033465, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "0071776f0787bda5e7657164", "case_id": "banking-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3686.45, "cost": 0.00172275, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2607980162968021, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ad3f24f3967c12a78129e1a2", "case_id": "banking-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2232.688, "cost": 0.00102525, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4df1651ab82cdea0260188a0", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2502.317, "cost": 0.000876, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "11f85b50d0bf8817d8b5bdb5", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2048.512, "cost": 0.0011835, "schemaValid": true, "pointHit": true, "boxIoU": 0.9855955978183417, "boxIoUAt50": true }, { "attempt_id": "93fcbe000911e5cd73ae586b", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2253.054, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.9636055492720064, "boxIoUAt50": true }, { "attempt_id": "5f8d6abb58a08fc8c14d03a3", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2513.501, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.19750444820274457, "boxIoUAt50": false }, { "attempt_id": "82ac6ec9b7988900f0f1e6b5", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1915.32, "cost": 0.0010905, "schemaValid": true, "pointHit": true, "boxIoU": 0.37178712606956044, "boxIoUAt50": false }, { "attempt_id": "5cb200aa9d7fc15a96cd5882", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 17472.849, "cost": 0.0085365, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "38426c101c215b869831a806", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 4309.198, "cost": 0.002304, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.9690805808019622, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "eec1e106b9f2f3204f57bd3c", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2412.342, "cost": 0.00105225, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0b8173feb4344b49893ef86e", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2486.145, "cost": 0.00117225, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7fd3eb57605ce802294a19e3", "case_id": "smart-home-android/clean/grounding/more", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1789.139, "cost": 0.0011775, "schemaValid": true, "pointHit": true, "boxIoU": 0.9682190384633694, "boxIoUAt50": true }, { "attempt_id": "19b06119fc77e8459350c26f", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2067.816, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.8644842570776636, "boxIoUAt50": true }, { "attempt_id": "e255a7ee449af790a9ddf15f", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2302.411, "cost": 0.00118125, "schemaValid": true, "pointHit": true, "boxIoU": 0.9773168444306137, "boxIoUAt50": true }, { "attempt_id": "5d28f5610c7c1ef137e51692", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2249.122, "cost": 0.0011805, "schemaValid": true, "pointHit": true, "boxIoU": 0.9636055492720064, "boxIoUAt50": true }, { "attempt_id": "5871e7fefe4bb986485fbe66", "case_id": "smart-home-android/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 8353.158, "cost": 0.00384525, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "4e862482fb0132f2fcdc1773", "case_id": "smart-home-android/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 8833.876, "cost": 0.0042465, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.16795329600000333, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bba2ca4bee8a4df7aaf0505d", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1839.821, "cost": 0.001071, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a9b35719e4662e862be9ad75", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2617.515, "cost": 0.00128175, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "x-ai/grok-4.7": { "name": "Grok 4.7", "classification_only": false, "point_source": "independently predicted", "planned_requests": 144, "completed_requests": 144, "transport_attempts": 144, "request_failures": 0, "invalid_responses": 2, "resolved_models": [ "x-ai/grok-4.7" ], "grounding": { "metrics": { "pointAccuracy": 0.7222222222222222, "meanIoU": 0.47746419320505673, "iouAt50": 0.5277777777777778, "schemaValidRate": 0.9861111111111112 }, "hits": 52, "count": 72, "consistency": [ { "case_id": "commerce-catalog-desktop/clean/grounding/search", "hit_consistent": true, "meanPairwisePointDistance": 9.333333333333334, "meanPairwiseBoxIoU": 0.5647596202785008 }, { "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "hit_consistent": false, "meanPairwisePointDistance": 2.23606797749979, "meanPairwiseBoxIoU": 0.9220570510245079 }, { "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "hit_consistent": true, "meanPairwisePointDistance": 7.113776245586032, "meanPairwiseBoxIoU": 0.3195541375184325 }, { "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "hit_consistent": false, "meanPairwisePointDistance": 137.2515360331866, "meanPairwiseBoxIoU": 0.3020292590844738 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "hit_consistent": true, "meanPairwisePointDistance": 2.192163740847158, "meanPairwiseBoxIoU": 0.7504315405426952 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "hit_consistent": true, "meanPairwisePointDistance": 2.2167605132909616, "meanPairwiseBoxIoU": 0.9620247683824585 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "hit_consistent": false, "meanPairwisePointDistance": 29.670454056443592, "meanPairwiseBoxIoU": 0.32296444001329344 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "hit_consistent": true, "meanPairwisePointDistance": 97.33333333333333, "meanPairwiseBoxIoU": 0.9011427816230518 }, { "case_id": "iot-control-desktop/clean/grounding/devices-nav", "hit_consistent": true, "meanPairwisePointDistance": 1.1380711874576983, "meanPairwiseBoxIoU": 0.9311503980581738 }, { "case_id": "iot-control-desktop/clean/grounding/restart-device", "hit_consistent": true, "meanPairwisePointDistance": 4.723362441819421, "meanPairwiseBoxIoU": 0.9078590153840412 }, { "case_id": "iot-control-desktop/clean/grounding/auto-mode", "hit_consistent": false, "meanPairwisePointDistance": 70.40202017303513, "meanPairwiseBoxIoU": 0.02837663534180947 }, { "case_id": "iot-control-desktop/clean/grounding/pump-card", "hit_consistent": true, "meanPairwisePointDistance": 2.6666666666666665, "meanPairwiseBoxIoU": 0.9165866089989946 }, { "case_id": "banking-ios/clean/grounding/profile", "hit_consistent": true, "meanPairwisePointDistance": 7.082638216102154, "meanPairwiseBoxIoU": 0.6846566571091296 }, { "case_id": "banking-ios/clean/grounding/transaction-search", "hit_consistent": true, "meanPairwisePointDistance": 1.3333333333333333, "meanPairwiseBoxIoU": 0.9398933647883184 }, { "case_id": "banking-ios/clean/grounding/hide-balance", "hit_consistent": false, "meanPairwisePointDistance": 101.16189670349267, "meanPairwiseBoxIoU": 0.11124769514443761 }, { "case_id": "banking-ios/clean/grounding/transfer", "hit_consistent": true, "meanPairwisePointDistance": 14, "meanPairwiseBoxIoU": 0.7069130824372759 }, { "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "hit_consistent": true, "meanPairwisePointDistance": 20, "meanPairwiseBoxIoU": 0.5467251973931836 }, { "case_id": "flight-pass-ios/clean/grounding/check-in", "hit_consistent": false, "meanPairwisePointDistance": 73.33333333333333, "meanPairwiseBoxIoU": 0.15860876469881469 }, { "case_id": "flight-pass-ios/clean/grounding/trips-tab", "hit_consistent": true, "meanPairwisePointDistance": 17.815508051599796, "meanPairwiseBoxIoU": 0.6392590966037126 }, { "case_id": "flight-pass-ios/clean/grounding/profile-tab", "hit_consistent": true, "meanPairwisePointDistance": 22.370951509168055, "meanPairwiseBoxIoU": 0.5181399129260249 }, { "case_id": "smart-home-android/clean/grounding/more", "hit_consistent": true, "meanPairwisePointDistance": 14.070360672800037, "meanPairwiseBoxIoU": 0.4933928380420715 }, { "case_id": "smart-home-android/clean/grounding/room-search", "hit_consistent": true, "meanPairwisePointDistance": 2, "meanPairwiseBoxIoU": 0.9182449494949495 }, { "case_id": "smart-home-android/clean/grounding/thermostat-card", "hit_consistent": false, "meanPairwisePointDistance": 34.666666666666664, "meanPairwiseBoxIoU": 0.4601898338364358 }, { "case_id": "smart-home-android/clean/grounding/add-device", "hit_consistent": false, "meanPairwisePointDistance": 38.666666666666664, "meanPairwiseBoxIoU": 0.3241514301570398 } ] }, "presence": { "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "correct": 36, "count": 36 }, "defects": { "metrics": { "balancedAccuracy": 0.9444444444444444, "recall": 0.9444444444444444, "specificity": 0.9444444444444444, "falsePositiveRate": 0.05555555555555558, "categoryAccuracy": 0.7222222222222222, "meanLocalizationIoU": 0.13043560474009513, "schemaValidRate": 0.9722222222222222 }, "correct": 34, "count": 36, "category_correct": 13, "category_count": 18, "region_supported": true }, "operations_by_track": { "grounding": { "count": 72, "transport_attempts": 72, "p50_ms": 16538.3215, "p95_ms": 28072.68995, "cost_usd": 0.6647400000000001 }, "element-presence": { "count": 36, "transport_attempts": 36, "p50_ms": 1967.0639999999999, "p95_ms": 5954.995500000001, "cost_usd": 0.10366800000000001 }, "layout-defect": { "count": 36, "transport_attempts": 36, "p50_ms": 13760.705, "p95_ms": 42030.323, "cost_usd": 0.34873800000000005 } }, "operations": { "p50_ms": 12026.273, "p95_ms": 28381.76634999999, "cost_usd": 1.1171459999999998, "missing_cost_count": 0 }, "attempts": [ { "attempt_id": "4cf284339e99c0b3e133c6fb", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 12823.326, "cost": 0.009178, "schemaValid": true, "pointHit": true, "boxIoU": 0.6123847491666653, "boxIoUAt50": true }, { "attempt_id": "a086d3bb2ebc5e8813dfcf1d", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 7110.748, "cost": 0.007036, "schemaValid": true, "pointHit": true, "boxIoU": 0.9298518774840181, "boxIoUAt50": true }, { "attempt_id": "1d8741ffb2721deb95b31455", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 9001.24, "cost": 0.00802, "schemaValid": true, "pointHit": true, "boxIoU": 0.8054616957215718, "boxIoUAt50": true }, { "attempt_id": "4e066cce9286edd5fed23742", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 28497.67, "cost": 0.014442, "schemaValid": true, "pointHit": true, "boxIoU": 0.4633584110504471, "boxIoUAt50": false }, { "attempt_id": "e9679de6c23148bef70f7f9d", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 12358.331, "cost": 0.009498, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "38da2f821e54f601efcc8f3c", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 3061.137, "cost": 0.005106, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "784ad5898b1cd2046f0d5072", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2393.78, "cost": 0.00415, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "920d7f32d5f9daa10eb20261", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1966.953, "cost": 0.004212, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "082a1b27a90e646ddfe6c24e", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 26897.021, "cost": 0.015302, "schemaValid": true, "pointHit": true, "boxIoU": 0.12570220819363287, "boxIoUAt50": false }, { "attempt_id": "0bf7ce39d0f1a20289eb003d", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 9617.432, "cost": 0.008176, "schemaValid": true, "pointHit": true, "boxIoU": 0.9532871304447741, "boxIoUAt50": true }, { "attempt_id": "e973d1df17c99525a8fa1ddd", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 11492.752, "cost": 0.009056, "schemaValid": true, "pointHit": true, "boxIoU": 0.9195319794122132, "boxIoUAt50": true }, { "attempt_id": "74350ac271aa3f750381021b", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 27514.819, "cost": 0.01723, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "d31fa327294e30ffab8324db", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 92188.226, "cost": 0.040602, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6ac6319c892e66399b0b3239", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 18337.152, "cost": 0.011214, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.22007947524747237, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9995c868ba288a33ed041232", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1536.113, "cost": 0.004204, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b4704da4f373f41daca4d75e", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 18388.066, "cost": 0.004282, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "092609151449d87a74eedc19", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 13266.724, "cost": 0.00937, "schemaValid": true, "pointHit": true, "boxIoU": 0.9393718895528291, "boxIoUAt50": true }, { "attempt_id": "f06c791c2e8c257533c0594b", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 15279.181, "cost": 0.011344, "schemaValid": true, "pointHit": true, "boxIoU": 0.8486264176387466, "boxIoUAt50": true }, { "attempt_id": "490f3984caff145db5d7a48f", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 25787.874, "cost": 0.01537, "schemaValid": true, "pointHit": true, "boxIoU": 0.5546031345451308, "boxIoUAt50": true }, { "attempt_id": "dd7b156e15ac5d6a85b377eb", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 14435.298, "cost": 0.010642, "schemaValid": true, "pointHit": true, "boxIoU": 0.9661717501012947, "boxIoUAt50": true }, { "attempt_id": "620c67c89e4b25aab66180db", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 6734.627, "cost": 0.006684, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "46fb8dfc5f0715bdc22fe4fe", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 25489.485, "cost": 0.015504, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07688298199502051, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c8a313ecf31b2e093ec641e7", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1876.169, "cost": 0.00412, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b32fc79265cad36f29013577", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1869.473, "cost": 0.004138, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9073d31620fce7272c3d496e", "case_id": "banking-ios/clean/grounding/profile", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 21933.828, "cost": 0.01171, "schemaValid": true, "pointHit": false, "boxIoU": 0.12431316755949481, "boxIoUAt50": false }, { "attempt_id": "a02aa2f8b9ac771ac621e352", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 10187.339, "cost": 0.005986, "schemaValid": true, "pointHit": true, "boxIoU": 0.7397226541295473, "boxIoUAt50": true }, { "attempt_id": "cdf05a2a0173a7ca4da713ba", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 53999.751, "cost": 0.025198, "schemaValid": true, "pointHit": true, "boxIoU": 0.2556944216775871, "boxIoUAt50": false }, { "attempt_id": "23718bf04fdb44ecf5f3a248", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 11305.16, "cost": 0.006242, "schemaValid": true, "pointHit": true, "boxIoU": 0.6058524638018783, "boxIoUAt50": true }, { "attempt_id": "af93f29f99aabfff7f196398", "case_id": "banking-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 8191.022, "cost": 0.0051, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2fe6584f54ba2f7ef0df0083", "case_id": "banking-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 20629.342, "cost": 0.009954, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.10944120978998978, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "360db29bbde2a92f9d361aa8", "case_id": "banking-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1663.076, "cost": 0.002144, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "54bebbfc50babed561614def", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1999.445, "cost": 0.002188, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c89b20dc7490d40953fe56dc", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 14677.909, "cost": 0.00933, "schemaValid": true, "pointHit": true, "boxIoU": 0.8020127136383285, "boxIoUAt50": true }, { "attempt_id": "c714dbf100333a6f05ee6b33", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 8914.7, "cost": 0.006118, "schemaValid": true, "pointHit": true, "boxIoU": 0.6500316398553241, "boxIoUAt50": true }, { "attempt_id": "3071dd5aaaa6b677bbc017e0", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 17816.095, "cost": 0.008964, "schemaValid": true, "pointHit": true, "boxIoU": 0.198611188186277, "boxIoUAt50": false }, { "attempt_id": "3dceb4df6909dbbdca165e4d", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 17094.407, "cost": 0.00898, "schemaValid": true, "pointHit": true, "boxIoU": 0.18151222880614112, "boxIoUAt50": false }, { "attempt_id": "7bafb8d4a7476297ee4ea397", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 12000.166, "cost": 0.007044, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8ab7c3f572c283b88055928d", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 19367.565, "cost": 0.010374, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "09ea304dc3f20d75f319da21", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1862.382, "cost": 0.002176, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "750423029549d1c078a251be", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1578.612, "cost": 0.002026, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ac87d6554c2067fe909b0e92", "case_id": "smart-home-android/clean/grounding/more", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 8748.56, "cost": 0.005628, "schemaValid": true, "pointHit": false, "boxIoU": 0.0030791810643134273, "boxIoUAt50": false }, { "attempt_id": "88d8caa9f20e41eb99d85a7d", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 9330.342, "cost": 0.005428, "schemaValid": true, "pointHit": true, "boxIoU": 0.7686541812176967, "boxIoUAt50": true }, { "attempt_id": "31d27f39e665ab1eb43c4cd0", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 13720.248, "cost": 0.008268, "schemaValid": true, "pointHit": true, "boxIoU": 0.8315053857768265, "boxIoUAt50": true }, { "attempt_id": "ba535ac948168ce0bb274236", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 8837.27, "cost": 0.005656, "schemaValid": true, "pointHit": false, "boxIoU": 0.18112505631715495, "boxIoUAt50": false }, { "attempt_id": "541ea8cc963492ebe1cd6106", "case_id": "smart-home-android/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3168.869, "cost": 0.00321, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "cdab20a097fe3f6364891a4f", "case_id": "smart-home-android/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 20278.101, "cost": 0.01023, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.19553987664226471, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ece09db922e6077ea9b1d82b", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2883.287, "cost": 0.00202, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "95aa9387e4a330b1370b2df8", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2517.11, "cost": 0.00237, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "71dc6792d615e4f4443baf0a", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 26801.477, "cost": 0.00964, "schemaValid": true, "pointHit": true, "boxIoU": 0.5903892649803613, "boxIoUAt50": true }, { "attempt_id": "125072d20b7bfa2d1717cece", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 7364.733, "cost": 0.007696, "schemaValid": true, "pointHit": true, "boxIoU": 0.9164451618426002, "boxIoUAt50": true }, { "attempt_id": "754b91dc9fc8d7893cddae18", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 20984.688, "cost": 0.012268, "schemaValid": true, "pointHit": true, "boxIoU": 0.792947086851935, "boxIoUAt50": true }, { "attempt_id": "e0b2f30300558704f1150ad5", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 17113.546, "cost": 0.009612, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "ec3eb0a4f864a63840a4688f", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 5627.872, "cost": 0.005928, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "999923042f6b32eca01f4444", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3071.424, "cost": 0.004452, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.24884953529185577, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7d9fb2772e28b3c15b2ce40e", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1725.572, "cost": 0.00412, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "02dbb104cc8a1d99e2c2bed8", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2082.602, "cost": 0.001956, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9a0f5e7cf4a0dac04d7e2288", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 18382.049, "cost": 0.010352, "schemaValid": true, "pointHit": true, "boxIoU": 0.11871875218287548, "boxIoUAt50": false }, { "attempt_id": "079e019976edc9281d88e550", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 16611.074, "cost": 0.00877, "schemaValid": true, "pointHit": true, "boxIoU": 0.9432559085245903, "boxIoUAt50": true }, { "attempt_id": "a34cf85ff6d339879b387b61", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 38770.375, "cost": 0.01778, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "99f9bd49ce54546649390f33", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 21242.113, "cost": 0.01246, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "ba36467c4ed2df3ac015069f", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 45428.24, "cost": 0.020652, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0ca8b09f8c67f80f6a9c249e", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 14960.406, "cost": 0.00963, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.30337259518020054, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ba71ec681cd5e8ffd890fb2b", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2222.047, "cost": 0.003958, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "95f994ee11e00696bfb9a126", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1967.175, "cost": 0.004282, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "68c28cbfe960ffb8e3b8ac1e", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 21098.743, "cost": 0.009472, "schemaValid": true, "pointHit": true, "boxIoU": 0.8971878071021419, "boxIoUAt50": true }, { "attempt_id": "fd4d892bd6a7b5885f10fa3f", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 11526.498, "cost": 0.006142, "schemaValid": true, "pointHit": true, "boxIoU": 0.931458660298867, "boxIoUAt50": true }, { "attempt_id": "fe2421ebcf1a732b9eb74ffc", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 16624.099, "cost": 0.010654, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "186d7694ee7b30536cef7a39", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 22313.33, "cost": 0.0133, "schemaValid": true, "pointHit": true, "boxIoU": 0.9661717501012947, "boxIoUAt50": true }, { "attempt_id": "39f1b2c6140b12f6e95b4937", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 13763.229, "cost": 0.009474, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0591c7a2be053bea0899294d", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 12815.072, "cost": 0.007182, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.08221276945586253, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "72102ce180e24a204ea019b3", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1782.289, "cost": 0.001786, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "709335f8e3362b6f39fd7616", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1886.514, "cost": 0.004138, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7848839492f7d7796d399181", "case_id": "banking-ios/clean/grounding/profile", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 20188.414, "cost": 0.009796, "schemaValid": true, "pointHit": false, "boxIoU": 0.007150811274864678, "boxIoUAt50": false }, { "attempt_id": "008059ab00da482c06497702", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 24961.06, "cost": 0.008722, "schemaValid": true, "pointHit": true, "boxIoU": 0.7244340971752056, "boxIoUAt50": true }, { "attempt_id": "f270d2b431c347a1776543ee", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 8558.186, "cost": 0.00565, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "a4c38f23f96de1e432695a43", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 8178.139, "cost": 0.004412, "schemaValid": true, "pointHit": true, "boxIoU": 0.6328178788415485, "boxIoUAt50": true }, { "attempt_id": "e88d31bb6d3c4c8163005301", "case_id": "banking-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 18845.408, "cost": 0.009876, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "073bef951846798a73f4b743", "case_id": "banking-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 20321.57, "cost": 0.00909, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.3897245822598261, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3df0c5db3216046a46cbb91d", "case_id": "banking-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1541.392, "cost": 0.002216, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "48da6128799e334f79f6811f", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1828.734, "cost": 0.001354, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0ccd0fa515b0b7d742fc8ac8", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 12052.38, "cost": 0.006042, "schemaValid": true, "pointHit": true, "boxIoU": 0.41004064163914394, "boxIoUAt50": false }, { "attempt_id": "19dd004a7b32cd909b598754", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 15236.237, "cost": 0.008128, "schemaValid": true, "pointHit": false, "boxIoU": 0.3249044422097997, "boxIoUAt50": false }, { "attempt_id": "d1b4b663f4b3aee4fc0b727e", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 23458.404, "cost": 0.011292, "schemaValid": true, "pointHit": true, "boxIoU": 0.17751327194056957, "boxIoUAt50": false }, { "attempt_id": "c66e042601e40193472ae8a4", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 25789.969, "cost": 0.009934, "schemaValid": true, "pointHit": true, "boxIoU": 0.21801261406062789, "boxIoUAt50": false }, { "attempt_id": "541ef74086f90cf331c55326", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 7640.781, "cost": 0.00489, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b1898ecd10e666dacfab6484", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 12451.428, "cost": 0.006246, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f232b5c71623f48312dc3abe", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1963.298, "cost": 0.00217, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0c6329b7c14cfbe9ce0db34f", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1771.474, "cost": 0.00202, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "34cfa062fd20b29640fc64c8", "case_id": "smart-home-android/clean/grounding/more", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 16245.896, "cost": 0.008424, "schemaValid": true, "pointHit": false, "boxIoU": 0.11497251065761899, "boxIoUAt50": false }, { "attempt_id": "6c6c62edc569c5417d154837", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 16465.569, "cost": 0.003052, "schemaValid": true, "pointHit": true, "boxIoU": 0.6754839774337335, "boxIoUAt50": true }, { "attempt_id": "70e66a1ab47fbd4da5288ce3", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 19155.392, "cost": 0.009834, "schemaValid": true, "pointHit": true, "boxIoU": 0.802856052190764, "boxIoUAt50": true }, { "attempt_id": "54768b4e12451f6d499c4eb6", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 9973.749, "cost": 0.00586, "schemaValid": true, "pointHit": true, "boxIoU": 0.5416210188348376, "boxIoUAt50": true }, { "attempt_id": "3ef0c85bb818914cb1ff3841", "case_id": "smart-home-android/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 15397.286, "cost": 0.00783, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "534f69e39e62b8dcf653f07e", "case_id": "smart-home-android/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 40897.684, "cost": 0.018066, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fe77f7c9410a1a29d0d0ea40", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2511.708, "cost": 0.002008, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c0660bb66d9a0e4c2e690a48", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 5929.425, "cost": 0.002448, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e73081551fddac0e2d740499", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 27724.979, "cost": 0.016612, "schemaValid": true, "pointHit": true, "boxIoU": 0.5844292731627003, "boxIoUAt50": true }, { "attempt_id": "cab9c39d1294b820d0b5b731", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 3, "track": "grounding", "status": "invalid_response", "wall_time_ms": 11214.551, "cost": 0.005752, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "302ce3204b30753ffdcdcd84", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 21569.789, "cost": 0.010312, "schemaValid": true, "pointHit": true, "boxIoU": 0.028187873097154863, "boxIoUAt50": false }, { "attempt_id": "d0697c74a04bc447905f18fe", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 17413.813, "cost": 0.011724, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "815939f0386fbe05795cf681", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 13758.181, "cost": 0.009168, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f5dddaf0a6854ba4389ccf1b", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3926.281, "cost": 0.004752, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.24198649480289722, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6539f030502149498002e468", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2189.448, "cost": 0.00412, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bb2f856fbb7820928858b21e", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2494.306, "cost": 0.001896, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "22b342cae89ba326ea1347c6", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 24179.799, "cost": 0.009338, "schemaValid": true, "pointHit": true, "boxIoU": 0.12862551536092665, "boxIoUAt50": false }, { "attempt_id": "8ecf20455b36ce3e205d1bcb", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 9668.776, "cost": 0.008086, "schemaValid": true, "pointHit": true, "boxIoU": 0.980933221573339, "boxIoUAt50": true }, { "attempt_id": "558a85ce99981dfbc752efe1", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 6946.037, "cost": 0.00437, "schemaValid": true, "pointHit": true, "boxIoU": 0.9388591616771313, "boxIoUAt50": true }, { "attempt_id": "0d32897e642d8dbe8d56ffd8", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 10417.926, "cost": 0.005542, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "2969012ede8e640d27e63a76", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 35633.502, "cost": 0.01671, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7f4cc82f7a6bd0d052914536", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 18817.981, "cost": 0.011376, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.13986262572040611, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c35e11eb965975439dcd169f", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 6031.707, "cost": 0.004012, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "38128a227234ef06ef91940d", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2555.458, "cost": 0.004426, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4c78f2e36544de9d08727451", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 13951.589, "cost": 0.006958, "schemaValid": true, "pointHit": true, "boxIoU": 0.9742778011647139, "boxIoUAt50": true }, { "attempt_id": "f640423a515bcd5bf060529b", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 8908.741, "cost": 0.005758, "schemaValid": true, "pointHit": true, "boxIoU": 0.9567052192648644, "boxIoUAt50": true }, { "attempt_id": "13737df8665a1811c7632608", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 18874.268, "cost": 0.009544, "schemaValid": true, "pointHit": true, "boxIoU": 0.05489210053566818, "boxIoUAt50": false }, { "attempt_id": "31956842634ccfd068ec17e7", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 19794.029, "cost": 0.011746, "schemaValid": true, "pointHit": true, "boxIoU": 0.8820396025366672, "boxIoUAt50": true }, { "attempt_id": "fe10a7c1575c203e93c5632c", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 13842.314, "cost": 0.00825, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9f89ea7a2cd2beb2f038bf24", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 5510.141, "cost": 0.005682, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.08065957790040756, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7de371530f2fca313ef87940", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1978.922, "cost": 0.004096, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4e51df3e75d34fa90f3d1fa2", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2083.845, "cost": 0.001834, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "54e59c401f8f1ce03e328979", "case_id": "banking-ios/clean/grounding/profile", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 17734.942, "cost": 0.009196, "schemaValid": true, "pointHit": false, "boxIoU": 0.08477496664561789, "boxIoUAt50": false }, { "attempt_id": "c20f775e04e2a25d8189ed22", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 17265.891, "cost": 0.005962, "schemaValid": true, "pointHit": true, "boxIoU": 0.7190635888810711, "boxIoUAt50": true }, { "attempt_id": "16e6d61f686b425660279512", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 21903.525, "cost": 0.01129, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "505fb423c71337da3da0328f", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 7906.255, "cost": 0.005288, "schemaValid": true, "pointHit": true, "boxIoU": 0.7784665296067191, "boxIoUAt50": true }, { "attempt_id": "30594a84e13e8abd25f847e9", "case_id": "banking-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 9425.826, "cost": 0.005676, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b0eb8780d1f059e9b5ae848e", "case_id": "banking-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 12395.518, "cost": 0.006306, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.09149114591726425, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "def3f9e4ae10d82809f33636", "case_id": "banking-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1414.539, "cost": 0.002174, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4311ea410df4b2989fdf6960", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2024.858, "cost": 0.002434, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "01f1c12b9a213e3d752992ff", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 12117.672, "cost": 0.006576, "schemaValid": true, "pointHit": true, "boxIoU": 0.9221865586237576, "boxIoUAt50": true }, { "attempt_id": "59adb18a7df21311c7c50d5f", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 17606.81, "cost": 0.008812, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "4ceabd8ac9bc41f0d55ba55e", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 18949.122, "cost": 0.008832, "schemaValid": true, "pointHit": true, "boxIoU": 0.1802427369324814, "boxIoUAt50": false }, { "attempt_id": "458ca376182e3351fc7b82e4", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 32806.785, "cost": 0.014848, "schemaValid": true, "pointHit": true, "boxIoU": 0.28135743626061854, "boxIoUAt50": false }, { "attempt_id": "90a86f38a87780e005e94052", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 10621.478, "cost": 0.006234, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "accd0492af7356bb4330eec4", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 17933.994, "cost": 0.009282, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5e5a0a4f340b00d25c07d204", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1455.145, "cost": 0.00124, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "01a6ed51b2df27faf5b9cc24", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1659.413, "cost": 0.00202, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "735a00783a7bfe73506acb97", "case_id": "smart-home-android/clean/grounding/more", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 4332.499, "cost": 0.00369, "schemaValid": true, "pointHit": false, "boxIoU": 0.12371008532257881, "boxIoUAt50": false }, { "attempt_id": "a1705a6a3acda29675a754bc", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 6477.193, "cost": 0.004588, "schemaValid": true, "pointHit": true, "boxIoU": 0.6965928517285377, "boxIoUAt50": true }, { "attempt_id": "c0ddcd15b89a8b576aa061b0", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 22650.793, "cost": 0.011652, "schemaValid": true, "pointHit": false, "boxIoU": 0.18937618765399833, "boxIoUAt50": false }, { "attempt_id": "c215bdb009d7042c1eff8888", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 9773.261, "cost": 0.006298, "schemaValid": true, "pointHit": true, "boxIoU": 0.6258539672502579, "boxIoUAt50": true }, { "attempt_id": "7e69cb2b58d96449d91d8e06", "case_id": "smart-home-android/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 10687.36, "cost": 0.005964, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c3f163eb213de3c911c118f4", "case_id": "smart-home-android/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 23642.072, "cost": 0.011502, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.16773801511824593, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1ec273939fc6d0343b6ef36c", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1569.737, "cost": 0.002164, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "463828cf322c4a57d1fa466e", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2545.245, "cost": 0.002766, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "qwen/qwen3.8-max-0902": { "name": "Qwen3.8 Max (0902)", "classification_only": false, "point_source": "independently predicted", "planned_requests": 144, "completed_requests": 144, "transport_attempts": 144, "request_failures": 0, "invalid_responses": 1, "resolved_models": [ "qwen/qwen3.8-max-0902" ], "grounding": { "metrics": { "pointAccuracy": 1, "meanIoU": 0.7602951915196607, "iouAt50": 0.8055555555555556, "schemaValidRate": 1 }, "hits": 72, "count": 72, "consistency": [ { "case_id": "commerce-catalog-desktop/clean/grounding/search", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.8915124189210569 }, { "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9669806384092098 }, { "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9915433025817064 }, { "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.8634920634920634 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.8953634085213033 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9859811549952395 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.8434065934065934 }, { "case_id": "iot-control-desktop/clean/grounding/devices-nav", "hit_consistent": true, "meanPairwisePointDistance": 26, "meanPairwiseBoxIoU": 0.9898797131621558 }, { "case_id": "iot-control-desktop/clean/grounding/restart-device", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9606060606060606 }, { "case_id": "iot-control-desktop/clean/grounding/auto-mode", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.8496825396825397 }, { "case_id": "iot-control-desktop/clean/grounding/pump-card", "hit_consistent": true, "meanPairwisePointDistance": 1.74535599249993, "meanPairwiseBoxIoU": 0.9916666666666667 }, { "case_id": "banking-ios/clean/grounding/profile", "hit_consistent": true, "meanPairwisePointDistance": 2.1081851067789197, "meanPairwiseBoxIoU": 0.9104899855433808 }, { "case_id": "banking-ios/clean/grounding/transaction-search", "hit_consistent": true, "meanPairwisePointDistance": 2, "meanPairwiseBoxIoU": 0.9992625368731564 }, { "case_id": "banking-ios/clean/grounding/hide-balance", "hit_consistent": true, "meanPairwisePointDistance": 0.9428090415820635, "meanPairwiseBoxIoU": 0.41276400367309457 }, { "case_id": "banking-ios/clean/grounding/transfer", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9645989974937343 }, { "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9588556565797971 }, { "case_id": "flight-pass-ios/clean/grounding/check-in", "hit_consistent": true, "meanPairwisePointDistance": 2, "meanPairwiseBoxIoU": 0.9761904761904763 }, { "case_id": "flight-pass-ios/clean/grounding/trips-tab", "hit_consistent": true, "meanPairwisePointDistance": 2.86851709182133, "meanPairwiseBoxIoU": 0.6306547619047619 }, { "case_id": "flight-pass-ios/clean/grounding/profile-tab", "hit_consistent": true, "meanPairwisePointDistance": 2.7748517734455866, "meanPairwiseBoxIoU": 0.7039042498719917 }, { "case_id": "smart-home-android/clean/grounding/more", "hit_consistent": true, "meanPairwisePointDistance": 2.3874258867227933, "meanPairwiseBoxIoU": 0.9203447655366057 }, { "case_id": "smart-home-android/clean/grounding/room-search", "hit_consistent": true, "meanPairwisePointDistance": 2.3874258867227933, "meanPairwiseBoxIoU": 0.9441528082266659 }, { "case_id": "smart-home-android/clean/grounding/thermostat-card", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9795619823403742 }, { "case_id": "smart-home-android/clean/grounding/add-device", "hit_consistent": true, "meanPairwisePointDistance": 2, "meanPairwiseBoxIoU": 0.9729275130156188 } ] }, "presence": { "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "correct": 36, "count": 36 }, "defects": { "metrics": { "balancedAccuracy": 0.9444444444444444, "recall": 0.9444444444444444, "specificity": 0.9444444444444444, "falsePositiveRate": 0.05555555555555558, "categoryAccuracy": 0.3888888888888889, "meanLocalizationIoU": 0.09093862991555518, "schemaValidRate": 0.9722222222222222 }, "correct": 34, "count": 36, "category_correct": 7, "category_count": 18, "region_supported": true }, "operations_by_track": { "grounding": { "count": 72, "transport_attempts": 72, "p50_ms": 8280.725, "p95_ms": 13386.899000000005, "cost_usd": 0.32538599999999995 }, "element-presence": { "count": 36, "transport_attempts": 36, "p50_ms": 4513.773499999999, "p95_ms": 6793.037, "cost_usd": 0.09615 }, "layout-defect": { "count": 36, "transport_attempts": 36, "p50_ms": 10978.5945, "p95_ms": 16472.862500000003, "cost_usd": 0.19424400000000006 } }, "operations": { "p50_ms": 8135.2695, "p95_ms": 14757.9275, "cost_usd": 0.6157799999999999, "missing_cost_count": 0 }, "attempts": [ { "attempt_id": "82fee94b9761275a18cd986e", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 6699.457, "cost": 0.004398, "schemaValid": true, "pointHit": true, "boxIoU": 0.8731130242600946, "boxIoUAt50": true }, { "attempt_id": "677721e50e15492021ac9b13", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 9031.751, "cost": 0.00564, "schemaValid": true, "pointHit": true, "boxIoU": 0.9831650708312063, "boxIoUAt50": true }, { "attempt_id": "a6e433ed93ea1b865b2608e2", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 9960.532, "cost": 0.00585, "schemaValid": true, "pointHit": true, "boxIoU": 0.8639985194638787, "boxIoUAt50": true }, { "attempt_id": "3eef128978c120f6b3baaa8e", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 7909.82, "cost": 0.005258, "schemaValid": true, "pointHit": true, "boxIoU": 0.06907206127460082, "boxIoUAt50": false }, { "attempt_id": "0fe8bcc441af22b3b8e25708", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 11014.058, "cost": 0.005886, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3bb77278df80c01139b10d8e", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 15272.9, "cost": 0.007614, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d2db648d63a44faef3a92dce", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 4243.984, "cost": 0.003414, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "eee5f9c4583ab4a549b4d70e", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 5157.428, "cost": 0.003776, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8878cced3acde7e8a668a299", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 14384.241, "cost": 0.007156, "schemaValid": true, "pointHit": true, "boxIoU": 0.8554940787117344, "boxIoUAt50": true }, { "attempt_id": "51ce51f2e46011c934550ec2", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 11237.6, "cost": 0.006132, "schemaValid": true, "pointHit": true, "boxIoU": 0.966053634410381, "boxIoUAt50": true }, { "attempt_id": "0df738ef8f715bb12f8e0e39", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 10903.341, "cost": 0.006154, "schemaValid": true, "pointHit": true, "boxIoU": 0.9717165373750636, "boxIoUAt50": true }, { "attempt_id": "62f23e362433a6087c08ca33", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 11416.06, "cost": 0.006318, "schemaValid": true, "pointHit": true, "boxIoU": 0.04211273134987978, "boxIoUAt50": false }, { "attempt_id": "6d8cc136d5d91f55c614cf3b", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 11642.491, "cost": 0.006018, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "80dd079d122e1b136313649b", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 14560.054, "cost": 0.00729, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "85ea87bbcfbfa99b82aecbd3", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 4221.5, "cost": 0.003522, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2d0df28a4d59a056967c9ac1", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 5516.807, "cost": 0.00378, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "64bb6a1e3465c4d9e1901b4b", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 11778.531, "cost": 0.006228, "schemaValid": true, "pointHit": true, "boxIoU": 0.9292554338240752, "boxIoUAt50": true }, { "attempt_id": "8de50ce0718ee4835d63db2e", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 11499.036, "cost": 0.00648, "schemaValid": true, "pointHit": true, "boxIoU": 0.9271299933442956, "boxIoUAt50": true }, { "attempt_id": "789925f279dc024c48d5ca8e", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 14372.664, "cost": 0.005568, "schemaValid": true, "pointHit": true, "boxIoU": 0.04990190957788016, "boxIoUAt50": false }, { "attempt_id": "034d6ebd041dadd40732bad9", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 8430.001, "cost": 0.004842, "schemaValid": true, "pointHit": true, "boxIoU": 0.9867897598688598, "boxIoUAt50": true }, { "attempt_id": "f1831cff64b3b0c302bc656b", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "invalid_response", "wall_time_ms": 10943.131, "cost": 0.005976, "schemaValid": false, "classificationCorrect": false, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": true, "falseNegative": false }, { "attempt_id": "38d951f3ef8aaf69c3a5ed3f", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 14069.58, "cost": 0.006642, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "93274c80cb2cf17d5e6d14d9", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 3825.236, "cost": 0.003384, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0f1967677598da9474d36cc8", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 6181.941, "cost": 0.003834, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f24e063299191cc7cdfc3c8c", "case_id": "banking-ios/clean/grounding/profile", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 9522.395, "cost": 0.003918, "schemaValid": true, "pointHit": true, "boxIoU": 0.8956988001152042, "boxIoUAt50": true }, { "attempt_id": "a03c334b4ae6162a36792fdb", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 6299.033, "cost": 0.002874, "schemaValid": true, "pointHit": true, "boxIoU": 0.8580530954386354, "boxIoUAt50": true }, { "attempt_id": "dc95e8f68fb42f2e6e7cf2c5", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 18694.646, "cost": 0.006834, "schemaValid": true, "pointHit": true, "boxIoU": 0.3766040675582008, "boxIoUAt50": false }, { "attempt_id": "1a3a656f527eb85ff5cb3570", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 11963.11, "cost": 0.00463, "schemaValid": true, "pointHit": true, "boxIoU": 0.9631500852905889, "boxIoUAt50": true }, { "attempt_id": "7a61d4027e9ed38095cb15bf", "case_id": "banking-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 9443.962, "cost": 0.003678, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dddd7866a0b31704746ecffb", "case_id": "banking-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 9555.309, "cost": 0.004128, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2468326953484393, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9f19ae3c3caa7587e8ef6b11", "case_id": "banking-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 3618.368, "cost": 0.001522, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5d39b6aa4d6f9a7a625a2ce3", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 4113.025, "cost": 0.00171, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1c349af4daa827921f662135", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 8075.348, "cost": 0.003356, "schemaValid": true, "pointHit": true, "boxIoU": 0.967717353638404, "boxIoUAt50": true }, { "attempt_id": "fb6a4abc8b99ff1d83e0e06e", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 6415.936, "cost": 0.002868, "schemaValid": true, "pointHit": true, "boxIoU": 0.9631500852905889, "boxIoUAt50": true }, { "attempt_id": "326a898019df7027820ff466", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 8575.622, "cost": 0.003416, "schemaValid": true, "pointHit": true, "boxIoU": 0.34539728620511767, "boxIoUAt50": false }, { "attempt_id": "1bed54386cc736c41eccf207", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 7646.073, "cost": 0.00339, "schemaValid": true, "pointHit": true, "boxIoU": 0.6802419429432598, "boxIoUAt50": true }, { "attempt_id": "7bede5e36919b85bfcb321c5", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 8784.771, "cost": 0.003624, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "89cd696e6e29f81e5ee03215", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 14776.697, "cost": 0.005346, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4b2933f73a869c21550a0736", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 3725.57, "cost": 0.001548, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "42eac65275efc7d8ad8f4017", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 4511.793, "cost": 0.0018, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8c2e1468adb4113d9b2b4927", "case_id": "smart-home-android/clean/grounding/more", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 9393.459, "cost": 0.003908, "schemaValid": true, "pointHit": true, "boxIoU": 0.8994141446785789, "boxIoUAt50": true }, { "attempt_id": "132c7b5b98759fa6755b39eb", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 5704.216, "cost": 0.002784, "schemaValid": true, "pointHit": true, "boxIoU": 0.8589733413484322, "boxIoUAt50": true }, { "attempt_id": "0f11e788b804b2503efbb288", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 7159.759, "cost": 0.003146, "schemaValid": true, "pointHit": true, "boxIoU": 0.9698606921846685, "boxIoUAt50": true }, { "attempt_id": "b7702bcefa95c4bfb7ee3ab5", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 6429.802, "cost": 0.00288, "schemaValid": true, "pointHit": true, "boxIoU": 0.9667670396505261, "boxIoUAt50": true }, { "attempt_id": "67706b9d383c05499ed49392", "case_id": "smart-home-android/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 9542.317, "cost": 0.00384, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3752ef06f09626e1e6c470fc", "case_id": "smart-home-android/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 14848.217, "cost": 0.005742, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dc809c8d14ffb6475f85f67f", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 3905.943, "cost": 0.001536, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "644364ce6639e5a2e4f3436a", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 4458.578, "cost": 0.001634, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "21922c639f3a4e5662bdf282", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 7870.502, "cost": 0.004932, "schemaValid": true, "pointHit": true, "boxIoU": 0.8201689429022837, "boxIoUAt50": true }, { "attempt_id": "4f27a0085ceb5317368b913b", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 10407.133, "cost": 0.006156, "schemaValid": true, "pointHit": true, "boxIoU": 0.9667045495306795, "boxIoUAt50": true }, { "attempt_id": "bd4d6956f7500d78be81dc3d", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 10062.688, "cost": 0.006012, "schemaValid": true, "pointHit": true, "boxIoU": 0.8639985194638787, "boxIoUAt50": true }, { "attempt_id": "3fecb2addc45c4a243712dbc", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 12580.364, "cost": 0.006554, "schemaValid": true, "pointHit": true, "boxIoU": 0.08634007659325102, "boxIoUAt50": false }, { "attempt_id": "9feec6e070afa7c6df3b1cf1", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 8638.671, "cost": 0.005376, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a562e69f6a631f6e275c44f1", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 13242.195, "cost": 0.006462, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.5492839183425455, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f683f329be4b1e040a07f398", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 3672.611, "cost": 0.003396, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f966fb6d40bb71a5cd755509", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 5863.918, "cost": 0.003764, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c5d8d24ee85b5d2f1eafe3de", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 11259.619, "cost": 0.006196, "schemaValid": true, "pointHit": true, "boxIoU": 0.727287978628368, "boxIoUAt50": true }, { "attempt_id": "68effeb9873de90c91b001bf", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 9962.512, "cost": 0.005916, "schemaValid": true, "pointHit": true, "boxIoU": 0.9798544006162435, "boxIoUAt50": true }, { "attempt_id": "64b15cd8ea0ff2f55a2ee7c1", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 9238.958, "cost": 0.005722, "schemaValid": true, "pointHit": true, "boxIoU": 0.9717165373750636, "boxIoUAt50": true }, { "attempt_id": "19e6cdc9a390ea0f942a70a7", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 12363.344, "cost": 0.006756, "schemaValid": true, "pointHit": true, "boxIoU": 0.05039130246994161, "boxIoUAt50": false }, { "attempt_id": "796b98edcb5a3c275885f4da", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 10559.956, "cost": 0.005814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0f62c129e717b22016c472a2", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 13441.034, "cost": 0.00708, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "90c9c6afbd82798ab28aa89b", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 4182.469, "cost": 0.003528, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5201fed75526062605edaed8", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 5565.307, "cost": 0.003762, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a195d498eb83f250260dcc9d", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 6570.448, "cost": 0.004092, "schemaValid": true, "pointHit": true, "boxIoU": 0.9363411900459144, "boxIoUAt50": true }, { "attempt_id": "a4542d2193b57e446ba1d67c", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 8098.455, "cost": 0.005202, "schemaValid": true, "pointHit": true, "boxIoU": 0.9544548666161667, "boxIoUAt50": true }, { "attempt_id": "8df299c784cde77c7540dfaf", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 10168.984, "cost": 0.005988, "schemaValid": true, "pointHit": true, "boxIoU": 0.05346633169058589, "boxIoUAt50": false }, { "attempt_id": "cc03534e196614c5b41c3b56", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 5961.283, "cost": 0.004608, "schemaValid": true, "pointHit": true, "boxIoU": 0.9853326491945819, "boxIoUAt50": true }, { "attempt_id": "71073156b03f7a1260823c0e", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 9941.267, "cost": 0.005136, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "45dc5c6efb7a082bfd05dac4", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 22098.189, "cost": 0.009258, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07472872658420113, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "624dbd6b2486dd63e8cc76cc", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 3753.97, "cost": 0.003426, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "eb792f20fc57f386517e6c4c", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 5033.956, "cost": 0.00384, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2e00177ff68c148b804c7b08", "case_id": "banking-ios/clean/grounding/profile", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 8604.139, "cost": 0.003774, "schemaValid": true, "pointHit": true, "boxIoU": 0.8623123831403234, "boxIoUAt50": true }, { "attempt_id": "01d87d8ba71fc28618619aca", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 8131.449, "cost": 0.00354, "schemaValid": true, "pointHit": true, "boxIoU": 0.8589733413484322, "boxIoUAt50": true }, { "attempt_id": "fd4a2842479bca5961c417a3", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 6743.26, "cost": 0.002742, "schemaValid": true, "pointHit": true, "boxIoU": 0.04824374976573555, "boxIoUAt50": false }, { "attempt_id": "cccd78cfe89d9d08ddf51904", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 6145.2, "cost": 0.002824, "schemaValid": true, "pointHit": true, "boxIoU": 0.9631500852905889, "boxIoUAt50": true }, { "attempt_id": "2d772375a682482738daa50b", "case_id": "banking-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 8600.657, "cost": 0.003678, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3ba5eee993b1ec519ff22701", "case_id": "banking-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 11658.208, "cost": 0.004782, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2679119317088936, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5f6a6a02263a92c5510709fd", "case_id": "banking-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 3965.815, "cost": 0.001564, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "063edad6b9f0b873fc37f3d6", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 6691.504, "cost": 0.002232, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "52fae29cf0c51bee4c7e069e", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 10936.124, "cost": 0.004064, "schemaValid": true, "pointHit": true, "boxIoU": 0.9709323614578007, "boxIoUAt50": true }, { "attempt_id": "442cb5703fecc030efb0297b", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 6756.472, "cost": 0.003168, "schemaValid": true, "pointHit": true, "boxIoU": 0.9631500852905889, "boxIoUAt50": true }, { "attempt_id": "911399daa6a15d70b4d25f6c", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 7426.774, "cost": 0.003212, "schemaValid": true, "pointHit": true, "boxIoU": 0.70094328996087, "boxIoUAt50": true }, { "attempt_id": "336e798b5736f42198768406", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 7584.935, "cost": 0.003318, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246387921363232, "boxIoUAt50": true }, { "attempt_id": "921c88935f359fbc624d21cb", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 9350.783, "cost": 0.003804, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6af4f68259c81ca763369b82", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 11780.09, "cost": 0.00483, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "64124fe7197103d02d89c3f3", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 3267.311, "cost": 0.001494, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5b098429bbe929ff1161087b", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 5391.019, "cost": 0.001788, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1b76c6f245eaa2522e6f9ca9", "case_id": "smart-home-android/clean/grounding/more", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 6057.446, "cost": 0.00275, "schemaValid": true, "pointHit": true, "boxIoU": 0.85965334174163, "boxIoUAt50": true }, { "attempt_id": "cc4ae1a11fdb0057b00ef570", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 6427.732, "cost": 0.002898, "schemaValid": true, "pointHit": true, "boxIoU": 0.8738416060860092, "boxIoUAt50": true }, { "attempt_id": "bdbccbb04fb11f7d0b80a200", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 6575.097, "cost": 0.002984, "schemaValid": true, "pointHit": true, "boxIoU": 0.9698606921846685, "boxIoUAt50": true }, { "attempt_id": "91c2e1e01753eeb19ccb5a96", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 6151.681, "cost": 0.002838, "schemaValid": true, "pointHit": true, "boxIoU": 0.9281264458254765, "boxIoUAt50": true }, { "attempt_id": "246a5e399c5ab211a4987160", "case_id": "smart-home-android/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 9393.389, "cost": 0.00384, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "71740367ed831dd0a8df5ab9", "case_id": "smart-home-android/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 17602.532, "cost": 0.006672, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "893d725a39995e326a689a83", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 4702.823, "cost": 0.00153, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f86c4fa88af9de89866bf82e", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 6135.654, "cost": 0.002354, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c6a47df6babf6574283695c9", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 9285.169, "cost": 0.005076, "schemaValid": true, "pointHit": true, "boxIoU": 0.9379110451424301, "boxIoUAt50": true }, { "attempt_id": "16a14cbf37303fe9f5316b98", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 10122.209, "cost": 0.005436, "schemaValid": true, "pointHit": true, "boxIoU": 0.9636546229691829, "boxIoUAt50": true }, { "attempt_id": "82cee9879a20b67dea9a7a1e", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 9571.345, "cost": 0.00507, "schemaValid": true, "pointHit": true, "boxIoU": 0.8603773835876081, "boxIoUAt50": true }, { "attempt_id": "b0493b1329297f3284367bb5", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 5475.456, "cost": 0.003698, "schemaValid": true, "pointHit": true, "boxIoU": 0.0805840714870343, "boxIoUAt50": false }, { "attempt_id": "a6326bc75fbbff3fabf96014", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 7690.259, "cost": 0.004338, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "be60c6b5b780b0499b5ffbfe", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 16096.306, "cost": 0.007404, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8cfd0e8592269faa44d9b379", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 4784.885, "cost": 0.003402, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9ce49db7adffac739c1e87c9", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 8139.09, "cost": 0.003776, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b7202172369a7ee564512dcc", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 7930.333, "cost": 0.005302, "schemaValid": true, "pointHit": true, "boxIoU": 0.8513044848441504, "boxIoUAt50": true }, { "attempt_id": "e28a2a28a92381bb9a687d9b", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 12243.318, "cost": 0.00666, "schemaValid": true, "pointHit": true, "boxIoU": 0.9867547837191748, "boxIoUAt50": true }, { "attempt_id": "6894f32ed423366c02c5bb99", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 15796.387, "cost": 0.007846, "schemaValid": true, "pointHit": true, "boxIoU": 0.9717165373750636, "boxIoUAt50": true }, { "attempt_id": "5e59cb87fb15d20dd0077fee", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 11005.823, "cost": 0.00636, "schemaValid": true, "pointHit": true, "boxIoU": 0.0388732904768121, "boxIoUAt50": false }, { "attempt_id": "fbb9b7b30495ce016d99736a", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 10265.132, "cost": 0.005868, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "05cd8518046a0e25b0db67f1", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 12464.158, "cost": 0.00669, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2125533685538191, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ddf8150bc0c316902ae0146e", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 7097.636, "cost": 0.003618, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b43d9c15a50661e6a6d3b9e1", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 4945.319, "cost": 0.003768, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8fc3d3d71523ad3e3ac4c093", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 5904.28, "cost": 0.004014, "schemaValid": true, "pointHit": true, "boxIoU": 0.9244480139486119, "boxIoUAt50": true }, { "attempt_id": "483cfa5c3c36fa4c8e5ea181", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 8959.919, "cost": 0.005616, "schemaValid": true, "pointHit": true, "boxIoU": 0.9271299933442956, "boxIoUAt50": true }, { "attempt_id": "dd117519aae1faa64e5d1c42", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 7807.681, "cost": 0.005028, "schemaValid": true, "pointHit": true, "boxIoU": 0.04170373871865699, "boxIoUAt50": false }, { "attempt_id": "390f9ff05f35d2f15edc506a", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 8014.302, "cost": 0.005148, "schemaValid": true, "pointHit": true, "boxIoU": 0.9853326491945819, "boxIoUAt50": true }, { "attempt_id": "0a2852dd469a4761a1d3f354", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 10178.28, "cost": 0.005874, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "459acda0beb0d3b88890a181", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 11952.815, "cost": 0.005868, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.0767553224151201, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "85436a3e00fc1ebc7b2e4688", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 3772.825, "cost": 0.003378, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b96d6b77914589888b8c583f", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 6031.972, "cost": 0.003924, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "69f0c34a68abfbc0fa4202a1", "case_id": "banking-ios/clean/grounding/profile", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 9359.378, "cost": 0.003894, "schemaValid": true, "pointHit": true, "boxIoU": 0.8762556191970973, "boxIoUAt50": true }, { "attempt_id": "c96c316abdc8d450d39935b0", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 6131.555, "cost": 0.002928, "schemaValid": true, "pointHit": true, "boxIoU": 0.8589733413484322, "boxIoUAt50": true }, { "attempt_id": "5f4b13d58173a6623b43fc1b", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 8053.891, "cost": 0.003492, "schemaValid": true, "pointHit": true, "boxIoU": 0.365000687284221, "boxIoUAt50": false }, { "attempt_id": "faee05f4f41550cee7d67546", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 6867.481, "cost": 0.002938, "schemaValid": true, "pointHit": true, "boxIoU": 0.9563075339073449, "boxIoUAt50": true }, { "attempt_id": "0fbb74538e3942bcfe3dc190", "case_id": "banking-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 9481.91, "cost": 0.003642, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9f90b0020b5f04d17392bdb9", "case_id": "banking-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 9283.46, "cost": 0.003942, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.20882937552697442, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7402aeb4543b6d10017d02bf", "case_id": "banking-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 4249.689, "cost": 0.001528, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "98772b09bce39fb2f509744c", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 4515.754, "cost": 0.001824, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "99fe2ecda1bb1d4ec230e9f1", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 7586.392, "cost": 0.00338, "schemaValid": true, "pointHit": true, "boxIoU": 0.9533566551249824, "boxIoUAt50": true }, { "attempt_id": "4052f4849f2e76ee19ad7d25", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 9318.454, "cost": 0.003834, "schemaValid": true, "pointHit": true, "boxIoU": 0.9631500852905889, "boxIoUAt50": true }, { "attempt_id": "8dcd135c48c5a8f6e80ec8df", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 6663.835, "cost": 0.00281, "schemaValid": true, "pointHit": true, "boxIoU": 0.36266715051537357, "boxIoUAt50": false }, { "attempt_id": "9b0485b4a991221257ab9790", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 8675.141, "cost": 0.003726, "schemaValid": true, "pointHit": true, "boxIoU": 0.6866855815943647, "boxIoUAt50": true }, { "attempt_id": "353ecb31d4373b2ec92e164a", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 8337.454, "cost": 0.003498, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3fa93130524ddff2b9c39125", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 14651.567, "cost": 0.005676, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bd07c93effecae1fac25c2dd", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 3981.59, "cost": 0.0015, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9d0bc3378b6956c0191177c7", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 4736.969, "cost": 0.0018, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ee1231c6c636a366187f673f", "case_id": "smart-home-android/clean/grounding/more", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 8921.406, "cost": 0.003776, "schemaValid": true, "pointHit": true, "boxIoU": 0.9230026241441964, "boxIoUAt50": true }, { "attempt_id": "4dd74ce288ada72d65fe45c9", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 6794.88, "cost": 0.00309, "schemaValid": true, "pointHit": true, "boxIoU": 0.8269549848518294, "boxIoUAt50": true }, { "attempt_id": "c796daadbb6165a8f78be250", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 7323.593, "cost": 0.003122, "schemaValid": true, "pointHit": true, "boxIoU": 0.988268257534494, "boxIoUAt50": true }, { "attempt_id": "a08f6df96d108556f94b4e9b", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 6502.589, "cost": 0.002868, "schemaValid": true, "pointHit": true, "boxIoU": 0.9281264458254765, "boxIoUAt50": true }, { "attempt_id": "d5c86723e664f0145ac9639f", "case_id": "smart-home-android/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 8801.264, "cost": 0.003396, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e1a30cf331884c9c129088da", "case_id": "smart-home-android/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 14145.417, "cost": 0.005532, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0d9b6e82af365549345b3534", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 3638.293, "cost": 0.00153, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8c8a86b48a878dff7702f6f6", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 5560.256, "cost": 0.001964, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "meta/muse-spark-1.3": { "name": "Muse Spark 1.3", "classification_only": false, "point_source": "independently predicted", "planned_requests": 144, "completed_requests": 144, "transport_attempts": 144, "request_failures": 0, "invalid_responses": 0, "resolved_models": [ "meta/muse-spark-1.3" ], "grounding": { "metrics": { "pointAccuracy": 0.9444444444444444, "meanIoU": 0.6303245836241916, "iouAt50": 0.6527777777777778, "schemaValidRate": 1 }, "hits": 68, "count": 72, "consistency": [ { "case_id": "commerce-catalog-desktop/clean/grounding/search", "hit_consistent": true, "meanPairwisePointDistance": 1.1380711874576983, "meanPairwiseBoxIoU": 0.9639238942426488 }, { "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9730612244897959 }, { "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9692594339841142 }, { "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.3888487149356714 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.8982456140350877 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "hit_consistent": true, "meanPairwisePointDistance": 0.9428090415820635, "meanPairwiseBoxIoU": 0.9915190254320377 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9855072463768115 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.3578104392620522 }, { "case_id": "iot-control-desktop/clean/grounding/devices-nav", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.986111111111111 }, { "case_id": "iot-control-desktop/clean/grounding/restart-device", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9869281045751634 }, { "case_id": "iot-control-desktop/clean/grounding/auto-mode", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9298245614035089 }, { "case_id": "iot-control-desktop/clean/grounding/pump-card", "hit_consistent": true, "meanPairwisePointDistance": 1.1380711874576983, "meanPairwiseBoxIoU": 0.9900866826740988 }, { "case_id": "banking-ios/clean/grounding/profile", "hit_consistent": true, "meanPairwisePointDistance": 1.6094757082487299, "meanPairwiseBoxIoU": 0.9134864208821284 }, { "case_id": "banking-ios/clean/grounding/transaction-search", "hit_consistent": true, "meanPairwisePointDistance": 5.333333333333333, "meanPairwiseBoxIoU": 0.857433663152272 }, { "case_id": "banking-ios/clean/grounding/hide-balance", "hit_consistent": false, "meanPairwisePointDistance": 19.005950484081264, "meanPairwiseBoxIoU": 0.22018348623853212 }, { "case_id": "banking-ios/clean/grounding/transfer", "hit_consistent": true, "meanPairwisePointDistance": 7.333333333333333, "meanPairwiseBoxIoU": 0.7768002606712284 }, { "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "hit_consistent": true, "meanPairwisePointDistance": 19.29245916182555, "meanPairwiseBoxIoU": 0.6274085098166924 }, { "case_id": "flight-pass-ios/clean/grounding/check-in", "hit_consistent": true, "meanPairwisePointDistance": 24.08451726060849, "meanPairwiseBoxIoU": 0.4749380523550188 }, { "case_id": "flight-pass-ios/clean/grounding/trips-tab", "hit_consistent": true, "meanPairwisePointDistance": 8.30444020739017, "meanPairwiseBoxIoU": 0.5345147187978659 }, { "case_id": "flight-pass-ios/clean/grounding/profile-tab", "hit_consistent": true, "meanPairwisePointDistance": 5.818397800120646, "meanPairwiseBoxIoU": 0.8084789549365113 }, { "case_id": "smart-home-android/clean/grounding/more", "hit_consistent": true, "meanPairwisePointDistance": 8.293439226684377, "meanPairwiseBoxIoU": 0.6901468834573445 }, { "case_id": "smart-home-android/clean/grounding/room-search", "hit_consistent": true, "meanPairwisePointDistance": 8, "meanPairwiseBoxIoU": 0.7693082879063177 }, { "case_id": "smart-home-android/clean/grounding/thermostat-card", "hit_consistent": true, "meanPairwisePointDistance": 17.333333333333332, "meanPairwiseBoxIoU": 0.6643592553385028 }, { "case_id": "smart-home-android/clean/grounding/add-device", "hit_consistent": true, "meanPairwisePointDistance": 10.666666666666666, "meanPairwiseBoxIoU": 0.7229731631905039 } ] }, "presence": { "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "correct": 36, "count": 36 }, "defects": { "metrics": { "balancedAccuracy": 0.8888888888888888, "recall": 0.7777777777777778, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.5555555555555556, "meanLocalizationIoU": 0.11872084096215486, "schemaValidRate": 1 }, "correct": 32, "count": 36, "category_correct": 10, "category_count": 18, "region_supported": true }, "operations_by_track": { "grounding": { "count": 72, "transport_attempts": 72, "p50_ms": 4514.503, "p95_ms": 8247.377450000002, "cost_usd": 0.22197255000000002 }, "element-presence": { "count": 36, "transport_attempts": 36, "p50_ms": 2059.679, "p95_ms": 4226.887, "cost_usd": 0.0725611 }, "layout-defect": { "count": 36, "transport_attempts": 36, "p50_ms": 5383.8775000000005, "p95_ms": 10127.82375, "cost_usd": 0.13964595000000002 } }, "operations": { "p50_ms": 4224.1025, "p95_ms": 9178.108499999998, "cost_usd": 0.4341795999999999, "missing_cost_count": 0 }, "attempts": [ { "attempt_id": "c4840eb58039aadc33627425", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 5130.329, "cost": 0.0038805, "schemaValid": true, "pointHit": true, "boxIoU": 0.9460288538850549, "boxIoUAt50": true }, { "attempt_id": "d99c6721e5244dc9b32327a8", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2394.253, "cost": 0.00296295, "schemaValid": true, "pointHit": true, "boxIoU": 0.9738672561034081, "boxIoUAt50": true }, { "attempt_id": "6cd30d65934f88b1095101bb", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3012.006, "cost": 0.0035065, "schemaValid": true, "pointHit": true, "boxIoU": 0.8390097400880562, "boxIoUAt50": true }, { "attempt_id": "120108d5e30c966acee0be40", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 6566.681, "cost": 0.00442575, "schemaValid": true, "pointHit": true, "boxIoU": 0.48983603453904406, "boxIoUAt50": false }, { "attempt_id": "885ca7b37478d313b5204b3c", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4094.989, "cost": 0.003715, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2da1c0bafa5a69428fcf1021", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 7804.679, "cost": 0.00570825, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.19620410390958695, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "41cab7512246e1f6e44ee590", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1660.368, "cost": 0.0026345, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1011288bad281f8100c4a688", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2793.606, "cost": 0.00302675, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a14876a2d6f8421de4f041ce", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 4958.411, "cost": 0.00385375, "schemaValid": true, "pointHit": true, "boxIoU": 0.727287978628368, "boxIoUAt50": true }, { "attempt_id": "3718dd6045498803b317df29", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 5077.906, "cost": 0.003583, "schemaValid": true, "pointHit": true, "boxIoU": 0.9912873371212171, "boxIoUAt50": true }, { "attempt_id": "cc16a8189e961a1781873611", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3093.514, "cost": 0.00336925, "schemaValid": true, "pointHit": true, "boxIoU": 0.9552124149315454, "boxIoUAt50": true }, { "attempt_id": "cdd9c58642e8e2682e6b9c03", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 4944.773, "cost": 0.00417375, "schemaValid": true, "pointHit": true, "boxIoU": 0.04211273134987978, "boxIoUAt50": false }, { "attempt_id": "dfd356026c95622b8dc691cd", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4069.602, "cost": 0.003749, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0dcfb7c3dade82e21a6c72d1", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 8221.94, "cost": 0.0055468, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.19976826029760092, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "862fae4152fcf8e2056cd64e", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2361.946, "cost": 0.003085, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "653f8ce8545d5232742d4e78", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1710.116, "cost": 0.00252825, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a8317b80f8c978e4a399570b", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3782.051, "cost": 0.003532, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "05c88460a6b99ae930eea12d", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3535.88, "cost": 0.0033535, "schemaValid": true, "pointHit": true, "boxIoU": 0.9445711627156905, "boxIoUAt50": true }, { "attempt_id": "70038dd12ef3d7e5d4b714d8", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 6483.177, "cost": 0.0046965, "schemaValid": true, "pointHit": true, "boxIoU": 0.04491171862009215, "boxIoUAt50": false }, { "attempt_id": "b28ede1db50d7e0768858352", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 5566.818, "cost": 0.0042965, "schemaValid": true, "pointHit": true, "boxIoU": 0.9624506509810044, "boxIoUAt50": true }, { "attempt_id": "26d4ad01b5b64895bb09b3dc", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 8450.704, "cost": 0.0032645, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1d424e6b7821bac134908a9d", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 5069.311, "cost": 0.004191, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "627195809acfdab94491f20d", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1947.442, "cost": 0.00268125, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "10e5f49edabc78bc1755ed0d", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1713.059, "cost": 0.00297025, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "148098451de77b3aa6a31c2a", "case_id": "banking-ios/clean/grounding/profile", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 6584.136, "cost": 0.002833, "schemaValid": true, "pointHit": true, "boxIoU": 0.5875607754294302, "boxIoUAt50": true }, { "attempt_id": "73737362109756cf8f2600dd", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3559.726, "cost": 0.002051, "schemaValid": true, "pointHit": true, "boxIoU": 0.5581523642063281, "boxIoUAt50": true }, { "attempt_id": "94204884e3cb033415483521", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 8765.75, "cost": 0.004142, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "0bc8b5f374521605a9324fc0", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3698.187, "cost": 0.0020965, "schemaValid": true, "pointHit": true, "boxIoU": 0.7789014146396906, "boxIoUAt50": true }, { "attempt_id": "d646955db59ccb86b4e4112f", "case_id": "banking-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4673.805, "cost": 0.0025655, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "30b5c588ae33820dc0562601", "case_id": "banking-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 5786.14, "cost": 0.003356, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2747085736458487, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "37c60aea4f2b2f999a72b7fb", "case_id": "banking-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 998.661, "cost": 0.000859, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8188e15a01e64cb24caaae25", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1719.045, "cost": 0.00127675, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "83e4c4151ec4e7f5e36b3833", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 4657.692, "cost": 0.00241725, "schemaValid": true, "pointHit": false, "boxIoU": 0.2968093854947569, "boxIoUAt50": false }, { "attempt_id": "a1b84b67c5bf9beb11bb2516", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3461.206, "cost": 0.00228475, "schemaValid": true, "pointHit": true, "boxIoU": 0.37540872824861743, "boxIoUAt50": false }, { "attempt_id": "a569acb110802d47beca5934", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 5154.227, "cost": 0.00262175, "schemaValid": true, "pointHit": true, "boxIoU": 0.1848186994688909, "boxIoUAt50": false }, { "attempt_id": "1cb3ddfb040c9f6e1a7d7662", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 4180.243, "cost": 0.00230175, "schemaValid": true, "pointHit": true, "boxIoU": 0.4029125354356231, "boxIoUAt50": false }, { "attempt_id": "eeaff4cbda5a6544ac9a2e2b", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4406.355, "cost": 0.00225525, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8289ecbb6cdc49691cd2e547", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 9527.423, "cost": 0.00474575, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "41f1f591cc567e7008171052", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2159.909, "cost": 0.00123425, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c02b1ce38c6fdff893b08a8b", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1534.092, "cost": 0.001247, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4a9a08a8871c36c403cc0c5b", "case_id": "smart-home-android/clean/grounding/more", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 4072.013, "cost": 0.002286, "schemaValid": true, "pointHit": true, "boxIoU": 0.4431933495364015, "boxIoUAt50": false }, { "attempt_id": "690e740f7186bbc667863ec4", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 5976.72, "cost": 0.0025865, "schemaValid": true, "pointHit": true, "boxIoU": 0.4668519164606063, "boxIoUAt50": false }, { "attempt_id": "ddd5e0222c39b6d7024f3e56", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 4820.662, "cost": 0.00235825, "schemaValid": true, "pointHit": true, "boxIoU": 0.7164376677428848, "boxIoUAt50": true }, { "attempt_id": "b463272350f1a16fb1b5e60c", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3709.935, "cost": 0.00217895, "schemaValid": true, "pointHit": true, "boxIoU": 0.8524161312754983, "boxIoUAt50": true }, { "attempt_id": "ace09f6fa24da577d2ad0aab", "case_id": "smart-home-android/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4540.406, "cost": 0.0025868, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "63bc2c1d8bda086c22a69c57", "case_id": "smart-home-android/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 8576.747, "cost": 0.0047373, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a2d6559c108f6076c44ab5ba", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1200.23, "cost": 0.00097925, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fb491a70e703988faecb1731", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 3738.898, "cost": 0.00155995, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6fedf8cf0bd7689598943848", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4146.282, "cost": 0.00413125, "schemaValid": true, "pointHit": true, "boxIoU": 0.9041122860814583, "boxIoUAt50": true }, { "attempt_id": "34b54d82aa745332da110ca0", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3722.337, "cost": 0.00373175, "schemaValid": true, "pointHit": true, "boxIoU": 0.9895046844489798, "boxIoUAt50": true }, { "attempt_id": "35a4cd360d00799023a8a81c", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2269.979, "cost": 0.003158, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "bfe06205e7d137899dc5c282", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 5519.213, "cost": 0.00393275, "schemaValid": true, "pointHit": true, "boxIoU": 0.08202307276358847, "boxIoUAt50": false }, { "attempt_id": "392ec1847412f65e3e81328f", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4061.871, "cost": 0.0035068, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "19a3d7e188ad2542835c4b9c", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 10107.222, "cost": 0.006333, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.208313492933077, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "38c8db4ae57870ee00b981cc", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1755.341, "cost": 0.002677, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5d6fef88230880e7b35187a4", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 3511.684, "cost": 0.0028355, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6b7516005cdda0718d8bcd8a", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4686.494, "cost": 0.0041555, "schemaValid": true, "pointHit": true, "boxIoU": 0.8554940787117344, "boxIoUAt50": true }, { "attempt_id": "0e2382ec5bdb0a47bff10859", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3864.761, "cost": 0.00343425, "schemaValid": true, "pointHit": true, "boxIoU": 0.9929150070415199, "boxIoUAt50": true }, { "attempt_id": "607fb2ad8aaf533a86ddfa3d", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4036.323, "cost": 0.003722, "schemaValid": true, "pointHit": true, "boxIoU": 0.9626820870964067, "boxIoUAt50": true }, { "attempt_id": "d5cb9a6c3344c1d808aac2be", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 7954.706, "cost": 0.00541095, "schemaValid": true, "pointHit": true, "boxIoU": 0.48803686865701, "boxIoUAt50": false }, { "attempt_id": "04d72f24500daf3c28d85aae", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4030.228, "cost": 0.00352375, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6b1afee8ff644724175c0682", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 9232.413, "cost": 0.00495605, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2000121812749165, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4a61c19918cba2f0f22d2e56", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2088.047, "cost": 0.00279175, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "767a14710e3e16f4a4671061", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1879.621, "cost": 0.0026685, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "56551b9575aa786b071ec44e", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4102.671, "cost": 0.00369775, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "6938cbbd378731124615fa90", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3189.235, "cost": 0.00343425, "schemaValid": true, "pointHit": true, "boxIoU": 0.9548654461447065, "boxIoUAt50": true }, { "attempt_id": "f14152a598c84ca53c300766", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 10004.12, "cost": 0.0040335, "schemaValid": true, "pointHit": true, "boxIoU": 0.04491171862009215, "boxIoUAt50": false }, { "attempt_id": "4aad829a07cd4d74b553f24b", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2779.843, "cost": 0.00380775, "schemaValid": true, "pointHit": true, "boxIoU": 0.9637247179241175, "boxIoUAt50": true }, { "attempt_id": "c7b5d886b365ab1899cc9078", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 5679.363, "cost": 0.00515575, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "194accc7de6aec2cf6f76fbe", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 8110.627, "cost": 0.004293, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "a25c00d2cad4504040fce81c", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1305.82, "cost": 0.00249, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "11ac2ba2c12707be419b9555", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1576.723, "cost": 0.00274925, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4d07de3c83aa0886cddb81fe", "case_id": "banking-ios/clean/grounding/profile", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4565.274, "cost": 0.00216195, "schemaValid": true, "pointHit": true, "boxIoU": 0.5308790465990185, "boxIoUAt50": true }, { "attempt_id": "9d0c37809db496076d907bed", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4251.86, "cost": 0.0020595, "schemaValid": true, "pointHit": true, "boxIoU": 0.5796870766859494, "boxIoUAt50": true }, { "attempt_id": "bfb5375d516a2b3fa97df37a", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 8605.087, "cost": 0.0037255, "schemaValid": true, "pointHit": true, "boxIoU": 0.3760286729779983, "boxIoUAt50": false }, { "attempt_id": "03ef85a070b90d98301965dc", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3435.201, "cost": 0.00190525, "schemaValid": true, "pointHit": true, "boxIoU": 0.5715447376161364, "boxIoUAt50": true }, { "attempt_id": "375b5c9d80d1cbff10255eba", "case_id": "banking-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3398.591, "cost": 0.002404, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c95b413b961a3342a3ce7613", "case_id": "banking-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 5499.644, "cost": 0.0028885, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.15950693319607853, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "96beff77ab400999ac9aa8b6", "case_id": "banking-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 8346.337, "cost": 0.00133925, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4c07887b5223302646da28d0", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1992.993, "cost": 0.0011337, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5933705c2414579691a3dd57", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4491.785, "cost": 0.00194125, "schemaValid": true, "pointHit": false, "boxIoU": 0.053292294820127, "boxIoUAt50": false }, { "attempt_id": "fb70fd9292dc1807b50ec483", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4145.82, "cost": 0.00219125, "schemaValid": true, "pointHit": true, "boxIoU": 0.7409360195642817, "boxIoUAt50": true }, { "attempt_id": "32cbf74032dcc49a3e4c2f00", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 6303.609, "cost": 0.0027705, "schemaValid": true, "pointHit": true, "boxIoU": 0.21713746188803806, "boxIoUAt50": false }, { "attempt_id": "bd1d2e9499cee934fcd3d1b1", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 5088.371, "cost": 0.00257845, "schemaValid": true, "pointHit": true, "boxIoU": 0.3772382544242191, "boxIoUAt50": false }, { "attempt_id": "008bd2a26622a5cb61cc1ca9", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4555.423, "cost": 0.00263775, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "989f2739ac656adc18132f6f", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 10189.629, "cost": 0.00446955, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e282c2b8c2ce877dd5d9bcc4", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2607.906, "cost": 0.0012895, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2437e1d6d346a17a2be549a7", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2096.625, "cost": 0.00120025, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "915cb2751652d892d199ce52", "case_id": "smart-home-android/clean/grounding/more", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 6584.864, "cost": 0.003289, "schemaValid": true, "pointHit": true, "boxIoU": 0.7412402995237796, "boxIoUAt50": true }, { "attempt_id": "184b2a54336eb2020be66f50", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4871.934, "cost": 0.0020897, "schemaValid": true, "pointHit": true, "boxIoU": 0.5037581538224538, "boxIoUAt50": true }, { "attempt_id": "ce1bc77f49b344862ad23345", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4451.012, "cost": 0.00222225, "schemaValid": true, "pointHit": true, "boxIoU": 0.8360179438372529, "boxIoUAt50": true }, { "attempt_id": "830d938feb40bd698f9c2c4c", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4537.221, "cost": 0.0022805, "schemaValid": true, "pointHit": true, "boxIoU": 0.6929247064547713, "boxIoUAt50": true }, { "attempt_id": "c275baf4c7bd81b099f6c4f4", "case_id": "smart-home-android/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4845.721, "cost": 0.0032965, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2b7051d12dad2deec7cafa54", "case_id": "smart-home-android/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 9623.897, "cost": 0.003832, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "2e7404ea0fd492cc999625e9", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 3867.526, "cost": 0.0015275, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "55c53309952637640eae5268", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1636.278, "cost": 0.0010825, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "19c4f343fc470ba701f09cd0", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 4919.335, "cost": 0.004042, "schemaValid": true, "pointHit": true, "boxIoU": 0.9460288538850549, "boxIoUAt50": true }, { "attempt_id": "a3946380b8e3e91100414759", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3589.196, "cost": 0.00326425, "schemaValid": true, "pointHit": true, "boxIoU": 0.9543436672283324, "boxIoUAt50": true }, { "attempt_id": "ec1979332084f38eeaf1cb1e", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3017.358, "cost": 0.00311975, "schemaValid": true, "pointHit": true, "boxIoU": 0.8211584690223529, "boxIoUAt50": true }, { "attempt_id": "8112750ceffc8699355da984", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 5153.009, "cost": 0.00405175, "schemaValid": true, "pointHit": true, "boxIoU": 0.07655486791268258, "boxIoUAt50": false }, { "attempt_id": "27225357f9b9dc1d9bcaa355", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 4520.658, "cost": 0.004038, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2318ede12b56a8297b7f8005", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 5268.111, "cost": 0.00449275, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.20295697488976272, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3865828e53faf187b7fe822a", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2902.658, "cost": 0.00289375, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "109d288239761e116dcd1632", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 4552.48, "cost": 0.00286525, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ccae9e2582bddbb8e64c31a7", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 5635.246, "cost": 0.00395575, "schemaValid": true, "pointHit": true, "boxIoU": 0.8127633351718062, "boxIoUAt50": true }, { "attempt_id": "199c2047378ccf9889d8f225", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3057.517, "cost": 0.00311975, "schemaValid": true, "pointHit": true, "boxIoU": 0.9935829124481744, "boxIoUAt50": true }, { "attempt_id": "cf298b4900dd78ea81b75ede", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3709.688, "cost": 0.00141605, "schemaValid": true, "pointHit": true, "boxIoU": 0.9626820870964067, "boxIoUAt50": true }, { "attempt_id": "8d20b9e8e708b619c96a41d1", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 5524.129, "cost": 0.00437775, "schemaValid": true, "pointHit": true, "boxIoU": 0.44196632516895223, "boxIoUAt50": false }, { "attempt_id": "0dd54005db99a74df0007d16", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3623.523, "cost": 0.003579, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "47a727d2739937dd8c707104", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 8870.383, "cost": 0.00571675, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.19976826029760092, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f545ac9adfdb035563b59ec3", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2071.34, "cost": 0.002711, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e18cda4b3fbe0fca258efb84", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1898.938, "cost": 0.002745, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "518864f6c21e17a743bc38c8", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3492.605, "cost": 0.003634, "schemaValid": true, "pointHit": true, "boxIoU": 0.9702012133248634, "boxIoUAt50": true }, { "attempt_id": "e95e2e3c627cec1dd6619043", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 4070.316, "cost": 0.003481, "schemaValid": true, "pointHit": true, "boxIoU": 0.9548654461447065, "boxIoUAt50": true }, { "attempt_id": "738383ad8fbeaba676d14e50", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 5215.849, "cost": 0.00377895, "schemaValid": true, "pointHit": true, "boxIoU": 0.04491171862009215, "boxIoUAt50": false }, { "attempt_id": "646569e26bfd7006e77cc8ef", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 5827.967, "cost": 0.00392295, "schemaValid": true, "pointHit": true, "boxIoU": 0.9759825860517736, "boxIoUAt50": true }, { "attempt_id": "3dc5339a2865effdf3996e98", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 4751.838, "cost": 0.0043355, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "45cb7fa576e429106f4ceaaf", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 7461.989, "cost": 0.0045948, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "158736f5db80bb430a7d4e31", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 4118.356, "cost": 0.00266425, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d6d716aa22b65ded88c51ba4", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2874.695, "cost": 0.0027025, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c71d0b004a56f185e44b982c", "case_id": "banking-ios/clean/grounding/profile", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3996.998, "cost": 0.00239525, "schemaValid": true, "pointHit": true, "boxIoU": 0.5266653486001348, "boxIoUAt50": true }, { "attempt_id": "4981c2d5183acb5b47a281fa", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 5635.72, "cost": 0.00213175, "schemaValid": true, "pointHit": true, "boxIoU": 0.7079767053520678, "boxIoUAt50": true }, { "attempt_id": "954e7574ee2f4d10189f420b", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 6211.79, "cost": 0.003105, "schemaValid": true, "pointHit": true, "boxIoU": 0.3432283042044735, "boxIoUAt50": false }, { "attempt_id": "d21bf025708af764bcf7b799", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3531.557, "cost": 0.001884, "schemaValid": true, "pointHit": true, "boxIoU": 0.5319893960660292, "boxIoUAt50": true }, { "attempt_id": "df3555f0caf687ae6e9eb542", "case_id": "banking-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 5255.672, "cost": 0.00262075, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "faf4598dabfc50f1a9e3b39d", "case_id": "banking-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 7262.092, "cost": 0.00349625, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2527743650603942, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1422c459a1942b7b1f1aebff", "case_id": "banking-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1335.989, "cost": 0.00097945, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e39047572fd9197398f8dc8d", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 3488.738, "cost": 0.001502, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a9140d464a54c6d3603dca53", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3759.576, "cost": 0.00226, "schemaValid": true, "pointHit": false, "boxIoU": 0.25853207920281157, "boxIoUAt50": false }, { "attempt_id": "3364ef95f6d4679266460613", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 4718.571, "cost": 0.002153, "schemaValid": true, "pointHit": true, "boxIoU": 0.8746573447238213, "boxIoUAt50": true }, { "attempt_id": "ea0e2545e0e4453c4c5a707a", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 4965.125, "cost": 0.00247725, "schemaValid": true, "pointHit": true, "boxIoU": 0.401288847063764, "boxIoUAt50": false }, { "attempt_id": "39c59af27c3cade9f2a26fc2", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 5649.775, "cost": 0.00270975, "schemaValid": true, "pointHit": true, "boxIoU": 0.4316920022524533, "boxIoUAt50": false }, { "attempt_id": "46652d740145a216ca4bc6df", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 5012.678, "cost": 0.00200025, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "908a94605c6844b04114cbca", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 5537.577, "cost": 0.00332205, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a5cc81f29541fd548f6cf23b", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2048.018, "cost": 0.00118325, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "be39707bc2d44f9f4e9492d7", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2989.859, "cost": 0.00171025, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9776fad24b9b683659f5108f", "case_id": "smart-home-android/clean/grounding/more", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 10731.201, "cost": 0.0028895, "schemaValid": true, "pointHit": true, "boxIoU": 0.5157967193372539, "boxIoUAt50": true }, { "attempt_id": "a377a735ba5d7fcb2c6c39d1", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 4375.623, "cost": 0.002034, "schemaValid": true, "pointHit": true, "boxIoU": 0.696738979870289, "boxIoUAt50": true }, { "attempt_id": "e8271dea05889ed3996b8751", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 4196.345, "cost": 0.0021462, "schemaValid": true, "pointHit": true, "boxIoU": 0.38605787087131604, "boxIoUAt50": false }, { "attempt_id": "9de307be859bf18952d237df", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3821.082, "cost": 0.00196645, "schemaValid": true, "pointHit": true, "boxIoU": 0.9084203931070349, "boxIoUAt50": true }, { "attempt_id": "ed69c79e8552c721158bb8dd", "case_id": "smart-home-android/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 4264.448, "cost": 0.00234025, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "935b17a1e4bde73db6c73fa4", "case_id": "smart-home-android/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 11757.833, "cost": 0.0053918, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2429619918139214, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "cea8c786d93358098f7a713b", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1937.974, "cost": 0.00131075, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "cf1329324bc5f61c7a52826e", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 2325.402, "cost": 0.00142675, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "openai/gpt-5.6-luna": { "name": "GPT-5.6 Luna", "classification_only": false, "point_source": "independently predicted", "planned_requests": 144, "completed_requests": 144, "transport_attempts": 144, "request_failures": 0, "invalid_responses": 0, "resolved_models": [ "openai/gpt-5.6-luna" ], "grounding": { "metrics": { "pointAccuracy": 0.9861111111111112, "meanIoU": 0.7857138245294218, "iouAt50": 0.7777777777777778, "schemaValidRate": 1 }, "hits": 71, "count": 72, "consistency": [ { "case_id": "commerce-catalog-desktop/clean/grounding/search", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9703703703703704 }, { "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "hit_consistent": true, "meanPairwisePointDistance": 0.4013864859597369, "meanPairwiseBoxIoU": 0.9847785977269706 }, { "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "hit_consistent": true, "meanPairwisePointDistance": 20.617788258100717, "meanPairwiseBoxIoU": 0.4334411859987299 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "hit_consistent": true, "meanPairwisePointDistance": 23.671427600007732, "meanPairwiseBoxIoU": 0.3789515073364203 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "hit_consistent": true, "meanPairwisePointDistance": 0.2000000000000076, "meanPairwiseBoxIoU": 0.9931071498424551 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "hit_consistent": true, "meanPairwisePointDistance": 0.21081851067790391, "meanPairwiseBoxIoU": 0.9928441025399243 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "hit_consistent": true, "meanPairwisePointDistance": 38.00589970490623, "meanPairwiseBoxIoU": 0.9615514407040647 }, { "case_id": "iot-control-desktop/clean/grounding/devices-nav", "hit_consistent": true, "meanPairwisePointDistance": 0.27487370837451347, "meanPairwiseBoxIoU": 0.984619274899643 }, { "case_id": "iot-control-desktop/clean/grounding/restart-device", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9866666666666667 }, { "case_id": "iot-control-desktop/clean/grounding/auto-mode", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.39295392953929537 }, { "case_id": "iot-control-desktop/clean/grounding/pump-card", "hit_consistent": true, "meanPairwisePointDistance": 4.060594014630335, "meanPairwiseBoxIoU": 0.9908227176220809 }, { "case_id": "banking-ios/clean/grounding/profile", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.966895900755124 }, { "case_id": "banking-ios/clean/grounding/transaction-search", "hit_consistent": false, "meanPairwisePointDistance": 272.4671647876054, "meanPairwiseBoxIoU": 0.9879996356313585 }, { "case_id": "banking-ios/clean/grounding/hide-balance", "hit_consistent": true, "meanPairwisePointDistance": 91.00612716941707, "meanPairwiseBoxIoU": 0.7543493036193069 }, { "case_id": "banking-ios/clean/grounding/transfer", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9770114942528737 }, { "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9977973568281939 }, { "case_id": "flight-pass-ios/clean/grounding/check-in", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "flight-pass-ios/clean/grounding/trips-tab", "hit_consistent": true, "meanPairwisePointDistance": 18.13658366003727, "meanPairwiseBoxIoU": 0.12146097316829024 }, { "case_id": "flight-pass-ios/clean/grounding/profile-tab", "hit_consistent": true, "meanPairwisePointDistance": 19.919612268759966, "meanPairwiseBoxIoU": 0.29964013135942963 }, { "case_id": "smart-home-android/clean/grounding/more", "hit_consistent": true, "meanPairwisePointDistance": 1.74535599249993, "meanPairwiseBoxIoU": 0.9727891156462585 }, { "case_id": "smart-home-android/clean/grounding/room-search", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 0.9869281045751634 }, { "case_id": "smart-home-android/clean/grounding/thermostat-card", "hit_consistent": true, "meanPairwisePointDistance": 0.6666666666666666, "meanPairwiseBoxIoU": 1 }, { "case_id": "smart-home-android/clean/grounding/add-device", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 } ] }, "presence": { "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "correct": 36, "count": 36 }, "defects": { "metrics": { "balancedAccuracy": 0.9166666666666667, "recall": 0.8333333333333334, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.6666666666666666, "meanLocalizationIoU": 0.23101988979596408, "schemaValidRate": 1 }, "correct": 33, "count": 36, "category_correct": 12, "category_count": 18, "region_supported": true }, "operations_by_track": { "grounding": { "count": 72, "transport_attempts": 72, "p50_ms": 2638.1845000000003, "p95_ms": 4710.214150000002, "cost_usd": 0.025831899999999998 }, "element-presence": { "count": 36, "transport_attempts": 36, "p50_ms": 1582.2415, "p95_ms": 2106.6175, "cost_usd": 0.00551861 }, "layout-defect": { "count": 36, "transport_attempts": 36, "p50_ms": 2863.331, "p95_ms": 4993.07775, "cost_usd": 0.012665639999999992 } }, "operations": { "p50_ms": 2340.68, "p95_ms": 4798.609049999999, "cost_usd": 0.04401615000000001, "missing_cost_count": 0 }, "attempts": [ { "attempt_id": "cfbefae03b4b98b6680a3941", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2557.562, "cost": 0.00063785, "schemaValid": true, "pointHit": true, "boxIoU": 0.9675787247276455, "boxIoUAt50": true }, { "attempt_id": "cf275bcbd6b314a019e51a89", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2002.377, "cost": 0.00059345, "schemaValid": true, "pointHit": true, "boxIoU": 0.9991253671871119, "boxIoUAt50": true }, { "attempt_id": "eea2091640a429b8cf09dfef", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2445.779, "cost": 0.00058865, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "2ac5a5a2aa407771b94ef0e3", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2889.489, "cost": 0.0006789, "schemaValid": true, "pointHit": true, "boxIoU": 0.3927264723920612, "boxIoUAt50": false }, { "attempt_id": "bf3e81e6c09c7c8c1360cf20", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 2031.334, "cost": 0.0005381, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8f46e3bb87249a2a43fa0ec8", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 4659.742, "cost": 0.0008669, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.19627488073125096, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "07bdda0a2e266e93915b491e", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1851.386, "cost": 0.0004394, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9363ae49e43ea2b4d989fb7b", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1728.628, "cost": 0.00043965, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7badf147541a100fe28d9029", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 5279.728, "cost": 0.000898, "schemaValid": true, "pointHit": true, "boxIoU": 0.9861414997178546, "boxIoUAt50": true }, { "attempt_id": "3c7521921e010400c3a2dc70", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2124.896, "cost": 0.00059705, "schemaValid": true, "pointHit": true, "boxIoU": 0.9828747723510349, "boxIoUAt50": true }, { "attempt_id": "d8bd60766e83186f505f90e2", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1872.078, "cost": 0.0005716, "schemaValid": true, "pointHit": true, "boxIoU": 0.9782124823785875, "boxIoUAt50": true }, { "attempt_id": "3eb61a105f716e4f8033d536", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3435.693, "cost": 0.00067145, "schemaValid": true, "pointHit": true, "boxIoU": 0.4151062757548743, "boxIoUAt50": false }, { "attempt_id": "b9daa247cb0b34dca9921941", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 1970.975, "cost": 0.0005501, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "87b38df661dbad999d8cca97", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3075.803, "cost": 0.0007169, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.2252895893430321, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6d5844200dbc75e9758cc760", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1310.359, "cost": 0.0004394, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fab0dccdac890a929c726dec", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 2286.694, "cost": 0.0004394, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a442489530322e16ac50b0a1", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2474.654, "cost": 0.00059345, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "911b73fb08cf534e974ded17", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2029.843, "cost": 0.00060305, "schemaValid": true, "pointHit": true, "boxIoU": 0.9722465288420963, "boxIoUAt50": true }, { "attempt_id": "56f48c076c0ffe19f79af063", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 4022.603, "cost": 0.00080585, "schemaValid": true, "pointHit": true, "boxIoU": 0.03724821107777484, "boxIoUAt50": false }, { "attempt_id": "4e4a8896469366fc4cbbaad4", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2636.063, "cost": 0.0006482, "schemaValid": true, "pointHit": true, "boxIoU": 0.9625187702800109, "boxIoUAt50": true }, { "attempt_id": "1fbfd7c25fc5a96482621630", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 1906.872, "cost": 0.0005501, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e32daf5bfa95240a47639045", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3786.785, "cost": 0.0006953, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "7de42af5998d76108495509f", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1545.792, "cost": 0.0004394, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "93da4381cadf3d8cc1685b9b", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1285.636, "cost": 0.0004394, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "715ec24e13484e1982d66447", "case_id": "banking-ios/clean/grounding/profile", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2815.331, "cost": 0.0003486, "schemaValid": true, "pointHit": true, "boxIoU": 0.983183943555824, "boxIoUAt50": true }, { "attempt_id": "94103708612d83859c550cd4", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1870.338, "cost": 0.0002718, "schemaValid": true, "pointHit": false, "boxIoU": 0.8624081314324273, "boxIoUAt50": true }, { "attempt_id": "a809e5221a509b4b4d330f7a", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 3775.462, "cost": 0.0004122, "schemaValid": true, "pointHit": true, "boxIoU": 0.24753546535747284, "boxIoUAt50": false }, { "attempt_id": "fc5b06b4e6519e55a5077fb9", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1693.212, "cost": 0.0001828, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "04ecff48e962f645a609dee6", "case_id": "banking-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 1952.685, "cost": 0.000234, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0e11177ee0f8896480a83828", "case_id": "banking-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 6577.935, "cost": 0.0008448, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.14847358485079123, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "59e6cac760821762c5e9a357", "case_id": "banking-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1643.7, "cost": 0.0001258, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "af1669d5e7dd22a69e594c4e", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1331.343, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "41c9a6733f24080965b3200a", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2096.359, "cost": 0.0002808, "schemaValid": true, "pointHit": true, "boxIoU": 0.9834378136487719, "boxIoUAt50": true }, { "attempt_id": "baf56a512502a3f3e552bc62", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2620.456, "cost": 0.0002622, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "3b4fd0a68edc780904bfcc41", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 4459.383, "cost": 0.0005202, "schemaValid": true, "pointHit": true, "boxIoU": 0.728029984198174, "boxIoUAt50": true }, { "attempt_id": "47c0c6144eb49dc8f586f224", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 5196.792, "cost": 0.0006318, "schemaValid": true, "pointHit": true, "boxIoU": 0.025953846438450525, "boxIoUAt50": false }, { "attempt_id": "17b5e8ead6b22982809ff545", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 1607.951, "cost": 0.0001824, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9dcd5bbebe849edd4736ccbc", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 3272.906, "cost": 0.0004164, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.6250531328258307, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "646eb044d863b0183e14ecfc", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1399.103, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6683d3dbb6d566b1570b5de9", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1142.449, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "66a15d65181733d4793d97e7", "case_id": "smart-home-android/clean/grounding/more", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2510.561, "cost": 0.0003188, "schemaValid": true, "pointHit": true, "boxIoU": 0.9519044965694483, "boxIoUAt50": true }, { "attempt_id": "d793cd430b4a1e096039a498", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 1953.374, "cost": 0.000273, "schemaValid": true, "pointHit": true, "boxIoU": 0.8476522453114688, "boxIoUAt50": true }, { "attempt_id": "01e7e05ba6c116ff23284041", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2741.614, "cost": 0.0002804, "schemaValid": true, "pointHit": true, "boxIoU": 0.9930474343914654, "boxIoUAt50": true }, { "attempt_id": "3bd3b39d956c1dd8a6583979", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 2441.413, "cost": 0.0002922, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "cf0ec898a08b6b67e7c136fc", "case_id": "smart-home-android/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 1877.584, "cost": 0.0001824, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8060942104921b9192fd3d2e", "case_id": "smart-home-android/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 5453.154, "cost": 0.0005688, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.16587704255301122, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5f991f4c13465907d427f6c7", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1574.431, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "86b756cd55b2fc7313576067", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 1571.391, "cost": 0.0001262, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4d9af248a2de6e7702a480aa", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2624.188, "cost": 0.00020398, "schemaValid": true, "pointHit": true, "boxIoU": 0.9675787247276455, "boxIoUAt50": true }, { "attempt_id": "2e2f087482a7c216f2e8ed74", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2639.215, "cost": 0.00020398, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774410293731403, "boxIoUAt50": true }, { "attempt_id": "68b02fd51c73ebd1a2d04add", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2374.357, "cost": 0.00015718, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "c219fcf3eacf7182468b931e", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3212.496, "cost": 0.0002628, "schemaValid": true, "pointHit": true, "boxIoU": 0.06561845821087078, "boxIoUAt50": false }, { "attempt_id": "5d60eef5600d721f2eabad4b", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1912.409, "cost": 0.00012592, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1249f1d43e64eedde43e8272", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4326.64, "cost": 0.00034192, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "cab340bd45229521fe8963fb", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2174.104, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "49f584920a2702083f8b2260", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1695.26, "cost": 0.00005118, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7033119c294bf2c2f1b1d09c", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 4499.557, "cost": 0.00038276, "schemaValid": true, "pointHit": true, "boxIoU": 0.9226695881695193, "boxIoUAt50": true }, { "attempt_id": "f588e96d861672539b1439da", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2674.183, "cost": 0.00020158, "schemaValid": true, "pointHit": true, "boxIoU": 0.9901169272881863, "boxIoUAt50": true }, { "attempt_id": "16e61c6286b5fe96f8966ffe", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2361.716, "cost": 0.00016076, "schemaValid": true, "pointHit": true, "boxIoU": 0.9782124823785875, "boxIoUAt50": true }, { "attempt_id": "154d15b5fae47c15a8eebc1a", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3044.124, "cost": 0.00026518, "schemaValid": true, "pointHit": true, "boxIoU": 0.4383777885332997, "boxIoUAt50": false }, { "attempt_id": "40dd2f1e817217363ce1ed14", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1838.894, "cost": 0.00007192, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ad91f373c3343239aa0ee3df", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2882.526, "cost": 0.00026272, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.18523152282204944, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "eb1a5016a4f8a9f594718219", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1567.365, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e105c79207b4230abaf1f4e5", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1663.235, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "88fdb61714508ee5ecc71b51", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3051.845, "cost": 0.00019558, "schemaValid": true, "pointHit": true, "boxIoU": 0.9759700544349836, "boxIoUAt50": true }, { "attempt_id": "3e3f04d2129fc96af5d7474f", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2248.721, "cost": 0.00017518, "schemaValid": true, "pointHit": true, "boxIoU": 0.9722465288420963, "boxIoUAt50": true }, { "attempt_id": "2c70658e05b66ec9115b25cb", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2797.365, "cost": 0.00020878, "schemaValid": true, "pointHit": true, "boxIoU": 0.03724821107777484, "boxIoUAt50": false }, { "attempt_id": "184bd51051935eb1f2ccd999", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3401.375, "cost": 0.00030724, "schemaValid": true, "pointHit": true, "boxIoU": 0.9757049108661706, "boxIoUAt50": true }, { "attempt_id": "4c252e2a57cb19098b9a6fe4", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1314.746, "cost": 0.00007192, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5696033d17a6d268a9cbc991", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2852.717, "cost": 0.00021472, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "e0d2c336d86c2970de18fc96", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2084.122, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d306a4872438a6da283fff26", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 2070.59, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8e613a2d9809a423adbdb4ec", "case_id": "banking-ios/clean/grounding/profile", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3145.509, "cost": 0.0003138, "schemaValid": true, "pointHit": true, "boxIoU": 0.9519044965694483, "boxIoUAt50": true }, { "attempt_id": "323802eedda9a1ffd965d145", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2165.251, "cost": 0.0002742, "schemaValid": true, "pointHit": true, "boxIoU": 0.8644842570776636, "boxIoUAt50": true }, { "attempt_id": "32b14d522d8a14827a7183a6", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3598.118, "cost": 0.0004602, "schemaValid": true, "pointHit": true, "boxIoU": 0.31091237762477114, "boxIoUAt50": false }, { "attempt_id": "b8b75894c68226036d7f1815", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2206.031, "cost": 0.0002812, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "085b6ab9a82795b1fa03f31a", "case_id": "banking-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1689.346, "cost": 0.0002232, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0c536e91cbd6bbde8f51b4aa", "case_id": "banking-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 4839.719, "cost": 0.0005928, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9bed864f0d904b403ace4685", "case_id": "banking-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1590.052, "cost": 0.0001258, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "30e2c72f345068b454923ba7", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1549.566, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0365bbab6da0c52ed53bab46", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3217.493, "cost": 0.0002904, "schemaValid": true, "pointHit": true, "boxIoU": 0.9834378136487719, "boxIoUAt50": true }, { "attempt_id": "2740e992687e2ee38975fb1c", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2062.139, "cost": 0.000261, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "93cb94529531a269155acc34", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 7159.776, "cost": 0.000789, "schemaValid": true, "pointHit": true, "boxIoU": 0.12041805387242056, "boxIoUAt50": false }, { "attempt_id": "2a27bd672b83039f7496bd36", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 3819.08, "cost": 0.0004146, "schemaValid": true, "pointHit": true, "boxIoU": 0.7402567985727962, "boxIoUAt50": true }, { "attempt_id": "c004ec6f33d06d99fb4fd71b", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1553.874, "cost": 0.0001824, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ef47af76b3b6aef4184d4d68", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 3369.157, "cost": 0.0003648, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.9779876941200134, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7dff1836c9ad0820cd7752a4", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1727.977, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "218164183a5514636b3414d4", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1637.673, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7e4cb37210e031518f08ab52", "case_id": "smart-home-android/clean/grounding/more", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2674.777, "cost": 0.0003092, "schemaValid": true, "pointHit": true, "boxIoU": 0.9386440009552793, "boxIoUAt50": true }, { "attempt_id": "865fdea8ffdc08d3d09523d1", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2301.625, "cost": 0.0002958, "schemaValid": true, "pointHit": true, "boxIoU": 0.8644842570776636, "boxIoUAt50": true }, { "attempt_id": "943aceb4a6348bc4b3d80f7b", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 2259.557, "cost": 0.0002828, "schemaValid": true, "pointHit": true, "boxIoU": 0.9930474343914654, "boxIoUAt50": true }, { "attempt_id": "d063e60905a2bd98509ba57b", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 1931.14, "cost": 0.000267, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "44ac71939ecc31a679255fc8", "case_id": "smart-home-android/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 1571.992, "cost": 0.0001824, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a8d9484ec36513a3ef2a8ce4", "case_id": "smart-home-android/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 2873.945, "cost": 0.0003852, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": 0, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "04d96cf034fa6adf16b5ab4c", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1170.105, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c8a3b6a702b8e53365323c43", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 1356.602, "cost": 0.0001262, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fabc19129ef2332b3671f649", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2607.252, "cost": 0.00020158, "schemaValid": true, "pointHit": true, "boxIoU": 0.9249255972888176, "boxIoUAt50": true }, { "attempt_id": "9832242fb3b84f850db0f819", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2880.584, "cost": 0.00020398, "schemaValid": true, "pointHit": true, "boxIoU": 0.9774410293731403, "boxIoUAt50": true }, { "attempt_id": "f5d784cdebd8b79bbf881eda", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 4310.854, "cost": 0.00016438, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "993eec5d98c643a991a5c912", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3362.197, "cost": 0.0002844, "schemaValid": true, "pointHit": true, "boxIoU": 0.3991789541161305, "boxIoUAt50": false }, { "attempt_id": "863160e3a00972d98c11617d", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1360.177, "cost": 0.00007192, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5b3aa6f5388bb66c0e3a1c5f", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3659.72, "cost": 0.00032272, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.220217130172021, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "508a7f91923de9bdb4a296a6", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1689.846, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "718ff6a4369bd1e783d15c45", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1802.623, "cost": 0.00005118, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1cbd4e8b54cfdf9be32f2093", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3613.909, "cost": 0.00028196, "schemaValid": true, "pointHit": true, "boxIoU": 0.09776838415060334, "boxIoUAt50": false }, { "attempt_id": "c330503de353bf9042689117", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2329.803, "cost": 0.00017398, "schemaValid": true, "pointHit": true, "boxIoU": 0.9828747723510349, "boxIoUAt50": true }, { "attempt_id": "62cbda902c1e51f422bb3831", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2092.604, "cost": 0.00017636, "schemaValid": true, "pointHit": true, "boxIoU": 0.9755949953274832, "boxIoUAt50": true }, { "attempt_id": "50f7d35e42c7784833612bb6", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3143.784, "cost": 0.00029878, "schemaValid": true, "pointHit": true, "boxIoU": 0.4165216317972209, "boxIoUAt50": false }, { "attempt_id": "ad42ce1000f3f2b4468d0261", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2331.863, "cost": 0.00007192, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e9ee133ec59b3db40b506dc2", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3519.574, "cost": 0.00028552, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.18580146596919422, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "73ac07a8c29d055f4274c935", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1605.636, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ade7592c1c8ae5d3dd8c799f", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1635.755, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9771a07de5cf366669fe854b", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2443.806, "cost": 0.00016918, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "80edbb2cf0764b694ea21816", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3001.649, "cost": 0.00020038, "schemaValid": true, "pointHit": true, "boxIoU": 0.983106476315828, "boxIoUAt50": true }, { "attempt_id": "882807ba051d195129d71d7d", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3801.498, "cost": 0.00033958, "schemaValid": true, "pointHit": true, "boxIoU": 0.4165027238696641, "boxIoUAt50": false }, { "attempt_id": "c60b0ba873eba27f6eff0dea", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3327.801, "cost": 0.00031684, "schemaValid": true, "pointHit": true, "boxIoU": 0.9637247179241175, "boxIoUAt50": true }, { "attempt_id": "7e20c7016c60ac06f597598f", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1574.31, "cost": 0.00007192, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "59afbb8a5831e5be81a05a10", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3652.272, "cost": 0.00026992, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.07793617352919888, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "395e50e4f7ba9deb08d473f2", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1816.096, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "724d41944b5dd8f5cfee2cf3", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1484.696, "cost": 0.00005116, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e206be579587c886906072ba", "case_id": "banking-ios/clean/grounding/profile", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2939.602, "cost": 0.0003342, "schemaValid": true, "pointHit": true, "boxIoU": 0.957664562561818, "boxIoUAt50": true }, { "attempt_id": "5906fa0c4a6b9a4938810e7a", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2498.69, "cost": 0.000333, "schemaValid": true, "pointHit": true, "boxIoU": 0.8644842570776636, "boxIoUAt50": true }, { "attempt_id": "836a7c65424e4fcda6710a0a", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 4967.684, "cost": 0.0005274, "schemaValid": true, "pointHit": true, "boxIoU": 0.35045101218351277, "boxIoUAt50": false }, { "attempt_id": "05fdaa03ce0c21ee07875395", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2549.451, "cost": 0.0002608, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "0236d845b2718d8f10c534df", "case_id": "banking-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 2964.066, "cost": 0.0002376, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a64e874930a1e12fa1d179d2", "case_id": "banking-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3817.183, "cost": 0.0005376, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6b07a333522e0b2596e4b49a", "case_id": "banking-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1262.795, "cost": 0.0001258, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ec67955a517287c85a000cf0", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1304.19, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "866b4365303060c5ef31f006", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2523.902, "cost": 0.0002964, "schemaValid": true, "pointHit": true, "boxIoU": 0.986342221925507, "boxIoUAt50": true }, { "attempt_id": "3aee81f4005fe26c0755c37f", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2236.501, "cost": 0.0002562, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "f84f135924caa02bdb285505", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 4278.606, "cost": 0.0005046, "schemaValid": true, "pointHit": true, "boxIoU": 0.03705170888382171, "boxIoUAt50": false }, { "attempt_id": "7f3ea9e9a2adb5abad8b851b", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 3219.206, "cost": 0.0003894, "schemaValid": true, "pointHit": true, "boxIoU": 0.7279606031598126, "boxIoUAt50": true }, { "attempt_id": "806f2c76d6b08d293541c829", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1696.029, "cost": 0.0002232, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d2a25ae55b6838e8aafeb2e9", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 3143.452, "cost": 0.0004092, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.9843387568579488, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c9c0e67fa4180cf07765a8f8", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1398.831, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "339160543ec748c35f4f6045", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1321.859, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5da3850901bc67f01312db53", "case_id": "smart-home-android/clean/grounding/more", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2349.497, "cost": 0.0003128, "schemaValid": true, "pointHit": true, "boxIoU": 0.9519044965694483, "boxIoUAt50": true }, { "attempt_id": "6c3d8af4b4e5a6848f545523", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2652.076, "cost": 0.0003258, "schemaValid": true, "pointHit": true, "boxIoU": 0.8476522453114688, "boxIoUAt50": true }, { "attempt_id": "a295de8ca80307d578153c18", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 2637.154, "cost": 0.000284, "schemaValid": true, "pointHit": true, "boxIoU": 0.9930474343914654, "boxIoUAt50": true }, { "attempt_id": "74f1afe63290d590cf4d5335", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 1916.182, "cost": 0.0002694, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "74ad1ec196c5cb4f9d532f86", "case_id": "smart-home-android/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1884.057, "cost": 0.0002268, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9688710eb31ae9637ff30e84", "case_id": "smart-home-android/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 4823.115, "cost": 0.0005712, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.16587704255301122, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0810a8b2eb2a2ca5af0727e5", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1189.769, "cost": 0.000126, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ebada7837471d38b2acf5e80", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1621.846, "cost": 0.0001262, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "typesafe/jev-1.14": { "name": "Jev 1.14", "classification_only": false, "point_source": "box center", "planned_requests": 144, "completed_requests": 144, "transport_attempts": 144, "request_failures": 0, "invalid_responses": 0, "resolved_models": [ "typesafe/jev-1.14-20261014" ], "grounding": { "metrics": { "pointAccuracy": 1, "meanIoU": 0.581164649155363, "iouAt50": 0.6944444444444444, "schemaValidRate": 1 }, "hits": 72, "count": 72, "consistency": [ { "case_id": "commerce-catalog-desktop/clean/grounding/search", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9999999999999978 }, { "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9999999999999997 }, { "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1.0000000000000007 }, { "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "hit_consistent": true, "meanPairwisePointDistance": 3.3333333333333335, "meanPairwiseBoxIoU": 0.6666666666666647 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9999999999999991 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "hit_consistent": true, "meanPairwisePointDistance": 3.3333333333333335, "meanPairwiseBoxIoU": 0.9814814814814813 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1.0000000000000002 }, { "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "hit_consistent": true, "meanPairwisePointDistance": 3.3333333333333335, "meanPairwiseBoxIoU": 0.7777777777777768 }, { "case_id": "iot-control-desktop/clean/grounding/devices-nav", "hit_consistent": true, "meanPairwisePointDistance": 5.6903559372884835, "meanPairwiseBoxIoU": 0.8452380952380952 }, { "case_id": "iot-control-desktop/clean/grounding/restart-device", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1.0000000000000007 }, { "case_id": "iot-control-desktop/clean/grounding/auto-mode", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 0.9999999999999963 }, { "case_id": "iot-control-desktop/clean/grounding/pump-card", "hit_consistent": true, "meanPairwisePointDistance": 3.3333333333333335, "meanPairwiseBoxIoU": 0.866666666666665 }, { "case_id": "banking-ios/clean/grounding/profile", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1.0000000000000002 }, { "case_id": "banking-ios/clean/grounding/transaction-search", "hit_consistent": true, "meanPairwisePointDistance": 3.3333333333333335, "meanPairwiseBoxIoU": 0.9926739926739927 }, { "case_id": "banking-ios/clean/grounding/hide-balance", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1.0000000000000016 }, { "case_id": "banking-ios/clean/grounding/transfer", "hit_consistent": true, "meanPairwisePointDistance": 3.3333333333333335, "meanPairwiseBoxIoU": 0.992673992673991 }, { "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "hit_consistent": true, "meanPairwisePointDistance": 6.666666666666667, "meanPairwiseBoxIoU": 0.9851019558884726 }, { "case_id": "flight-pass-ios/clean/grounding/check-in", "hit_consistent": true, "meanPairwisePointDistance": 3.3333333333333335, "meanPairwiseBoxIoU": 0.916666666666666 }, { "case_id": "flight-pass-ios/clean/grounding/trips-tab", "hit_consistent": true, "meanPairwisePointDistance": 5.6903559372884915, "meanPairwiseBoxIoU": 0.8660561660561671 }, { "case_id": "flight-pass-ios/clean/grounding/profile-tab", "hit_consistent": true, "meanPairwisePointDistance": 4.714045207910317, "meanPairwiseBoxIoU": 0.8580246913580251 }, { "case_id": "smart-home-android/clean/grounding/more", "hit_consistent": true, "meanPairwisePointDistance": 7.750469233121475, "meanPairwiseBoxIoU": 0.7248677248677254 }, { "case_id": "smart-home-android/clean/grounding/room-search", "hit_consistent": true, "meanPairwisePointDistance": 3.3333333333333335, "meanPairwiseBoxIoU": 0.8888888888888893 }, { "case_id": "smart-home-android/clean/grounding/thermostat-card", "hit_consistent": true, "meanPairwisePointDistance": 0, "meanPairwiseBoxIoU": 1 }, { "case_id": "smart-home-android/clean/grounding/add-device", "hit_consistent": true, "meanPairwisePointDistance": 3.3333333333333335, "meanPairwiseBoxIoU": 0.9166666666666652 } ] }, "presence": { "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "correct": 36, "count": 36 }, "defects": { "metrics": { "balancedAccuracy": 1, "recall": 1, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.6666666666666666, "meanLocalizationIoU": 0.20766944054299674, "schemaValidRate": 1 }, "correct": 36, "count": 36, "category_correct": 12, "category_count": 18, "region_supported": true }, "operations_by_track": { "grounding": { "count": 72, "transport_attempts": 72, "p50_ms": 351.6645, "p95_ms": 479.81985000000003, "cost_usd": 0.00785358 }, "element-presence": { "count": 36, "transport_attempts": 36, "p50_ms": 368.1975, "p95_ms": 479.40150000000006, "cost_usd": 0.0038481240000000022 }, "layout-defect": { "count": 36, "transport_attempts": 36, "p50_ms": 371.1345, "p95_ms": 481.73, "cost_usd": 0.00419853 } }, "operations": { "p50_ms": 365.1305, "p95_ms": 482.7922, "cost_usd": 0.015900234, "missing_cost_count": 0 }, "attempts": [ { "attempt_id": "e0d5e469aebbc478e0b31c83", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 429.851, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.08281870393145319, "boxIoUAt50": false }, { "attempt_id": "b17039092d520e3e8b29919a", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 406.172, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.9114781207715029, "boxIoUAt50": true }, { "attempt_id": "81b4e96ca7cbf94ed3c25057", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 395.283, "cost": 0.000163002, "schemaValid": true, "pointHit": true, "boxIoU": 0.6520571060405461, "boxIoUAt50": true }, { "attempt_id": "b678f6e33c4b696a850e04d5", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 380.915, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.057593616762300605, "boxIoUAt50": false }, { "attempt_id": "ae2fff48c521e4bc41678c86", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 471.999, "cost": 0.00017052, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2b6eca1911fff379e529c041", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 387.371, "cost": 0.000170394, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "46f0d319594a238023205548", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 353.003, "cost": 0.000160692, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "51a4f3e532ccf2523b31c7fb", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 440.878, "cost": 0.00016065, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a1f4bef065acc03b97cad15d", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 525.288, "cost": 0.000162834, "schemaValid": true, "pointHit": true, "boxIoU": 0.5141854196665914, "boxIoUAt50": true }, { "attempt_id": "3f18951326436c99a7e0c0eb", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 365.579, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.9225420197089684, "boxIoUAt50": true }, { "attempt_id": "6e9abf33bc8c2049292a84fb", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 378.359, "cost": 0.000162918, "schemaValid": true, "pointHit": true, "boxIoU": 0.7380562667739121, "boxIoUAt50": true }, { "attempt_id": "836fd68f4f25947e6763fc02", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 335.277, "cost": 0.000162792, "schemaValid": true, "pointHit": true, "boxIoU": 0.11066119725816245, "boxIoUAt50": false }, { "attempt_id": "8cac365e4016a130f4712648", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 484.796, "cost": 0.00017052, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "eed341b4b7397bc4343dbb83", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 363.893, "cost": 0.000170436, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.7290774308740957, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0976cbdf4fe37f2ce8cc39e5", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 331.451, "cost": 0.000160734, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "359d0b9051e9fb5ce4a9d489", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 388.972, "cost": 0.000160818, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "50066dbe03016973c2b5b291", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 340.889, "cost": 0.00016296, "schemaValid": true, "pointHit": true, "boxIoU": 0.87962996164, "boxIoUAt50": true }, { "attempt_id": "83f4202a336eac93d713a5e6", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 324.021, "cost": 0.00016296, "schemaValid": true, "pointHit": true, "boxIoU": 0.851017023045835, "boxIoUAt50": true }, { "attempt_id": "7f1de637f55b920dabb08358", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 405.007, "cost": 0.000162834, "schemaValid": true, "pointHit": true, "boxIoU": 0.03564422112705726, "boxIoUAt50": false }, { "attempt_id": "c668405b8224047df399ce43", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 368.048, "cost": 0.000163002, "schemaValid": true, "pointHit": true, "boxIoU": 0.4914893826165678, "boxIoUAt50": false }, { "attempt_id": "8704189ea3b33b19852fc96b", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 480.708, "cost": 0.000170394, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e8195bb60d47151ae60ded7c", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 375.877, "cost": 0.00017052, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.1148936219103669, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "228ee8d5e40768d8bb5d4c01", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 345.873, "cost": 0.00016065, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dc84f61f76341c323f6dccc4", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 459.201, "cost": 0.000160776, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "708daae5ab50ae546a1cc4ac", "case_id": "banking-ios/clean/grounding/profile", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 393.884, "cost": 0.000055272, "schemaValid": true, "pointHit": true, "boxIoU": 0.7406094704642425, "boxIoUAt50": true }, { "attempt_id": "0585b8fe2fbf16b5caf6fcd1", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 477.087, "cost": 0.00005502, "schemaValid": true, "pointHit": true, "boxIoU": 0.6314275263134961, "boxIoUAt50": true }, { "attempt_id": "9057a7ad26dd7ba0f6630918", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 413.333, "cost": 0.00005523, "schemaValid": true, "pointHit": true, "boxIoU": 0.030718705711512147, "boxIoUAt50": false }, { "attempt_id": "7053369b0ef3a99560fb7f91", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 434.012, "cost": 0.00005523, "schemaValid": true, "pointHit": true, "boxIoU": 0.8042013113136398, "boxIoUAt50": true }, { "attempt_id": "14d3c527b99436d9d490f10d", "case_id": "banking-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 374.696, "cost": 0.000062874, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a1ac2f7b2327bf5a491c12c7", "case_id": "banking-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 296.645, "cost": 0.000062832, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.17883060587496785, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f7c584d655bd822954ce35ed", "case_id": "banking-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 371.713, "cost": 0.000053046, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1487ec04d1d6e295f0c07d62", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 379.595, "cost": 0.000053004, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "80b8ab259400a52aa00b36ea", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 310.938, "cost": 0.000055398, "schemaValid": true, "pointHit": true, "boxIoU": 0.9405581814225531, "boxIoUAt50": true }, { "attempt_id": "a6b609a0093dc33ddf6674ec", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 475.107, "cost": 0.00005523, "schemaValid": true, "pointHit": true, "boxIoU": 0.8042013113136398, "boxIoUAt50": true }, { "attempt_id": "e90eddfc4b6b55b20b18ff14", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 338.494, "cost": 0.00005523, "schemaValid": true, "pointHit": true, "boxIoU": 0.38626538952014466, "boxIoUAt50": false }, { "attempt_id": "dc095bd22f4d3443d671c043", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 321.849, "cost": 0.000055272, "schemaValid": true, "pointHit": true, "boxIoU": 0.44477357807828505, "boxIoUAt50": false }, { "attempt_id": "d8921cb8ff0e3738a889d41e", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 301.635, "cost": 0.000062706, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e7d5deef51beec54c6bce5cf", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 369.2, "cost": 0.000062748, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2d84640971f72ba6311df362", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 284.437, "cost": 0.00005313, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e810929d4507030c402a2d90", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 379.364, "cost": 0.000053172, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "84e6bebd7ede8de5f2bdaef3", "case_id": "smart-home-android/clean/grounding/more", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 355.16, "cost": 0.000055272, "schemaValid": true, "pointHit": true, "boxIoU": 0.5554571028481818, "boxIoUAt50": true }, { "attempt_id": "2949352509b83be960c0b5cb", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 336.563, "cost": 0.000055188, "schemaValid": true, "pointHit": true, "boxIoU": 0.6249101412643956, "boxIoUAt50": true }, { "attempt_id": "63672e60e922d55f0932fe0c", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 374.672, "cost": 0.000055146, "schemaValid": true, "pointHit": true, "boxIoU": 0.8415755762538395, "boxIoUAt50": true }, { "attempt_id": "2f540c3b811d9f33a7cbbfca", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 1, "track": "grounding", "status": "success", "wall_time_ms": 333.075, "cost": 0.000055356, "schemaValid": true, "pointHit": true, "boxIoU": 0.7039772890411281, "boxIoUAt50": true }, { "attempt_id": "4c5d032dba3ed8fb9fca2546", "case_id": "smart-home-android/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 427.309, "cost": 0.000062958, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "801d6c9e084798dbf1025491", "case_id": "smart-home-android/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 394.229, "cost": 0.000062748, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.23646553351479624, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8a3940d33422f20a05a082fd", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 379.292, "cost": 0.000053088, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "eab55827358246aa61f1bca3", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 410.831, "cost": 0.00005313, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "23b89846517f83ef44ba25f2", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 441.655, "cost": 0.000162918, "schemaValid": true, "pointHit": true, "boxIoU": 0.08281870393145319, "boxIoUAt50": false }, { "attempt_id": "f17c7c699a2938bbb91f44ae", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 422.987, "cost": 0.000162792, "schemaValid": true, "pointHit": true, "boxIoU": 0.9114781207715029, "boxIoUAt50": true }, { "attempt_id": "ef8fac431fc9e95d37f53f4f", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 378.254, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.6520571060405461, "boxIoUAt50": true }, { "attempt_id": "60943757e05d98fb83d96b19", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 419.768, "cost": 0.00016296, "schemaValid": true, "pointHit": true, "boxIoU": 0.057593616762300605, "boxIoUAt50": false }, { "attempt_id": "d529892e761773992f998d7f", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 379.616, "cost": 0.00017052, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "58b97a1a50d20a8ee194605a", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 390.836, "cost": 0.000170436, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "04789ba3b43d2d5cb33486f2", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 382.781, "cost": 0.000160818, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6108284d335206e9a839ef78", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 540.003, "cost": 0.00016065, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dbb4fcf1f7234614bef0921b", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 410.183, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.5141854196665914, "boxIoUAt50": true }, { "attempt_id": "94e3f368f7cac0ef447ab730", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 660.452, "cost": 0.00016296, "schemaValid": true, "pointHit": true, "boxIoU": 0.9225420197089684, "boxIoUAt50": true }, { "attempt_id": "53bbc72b68451f6d63a02b5c", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 483.16, "cost": 0.000162918, "schemaValid": true, "pointHit": true, "boxIoU": 0.7380562667739121, "boxIoUAt50": true }, { "attempt_id": "74bf2ca1eeafe6d4dac3ffc8", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 374.137, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.16360954112088283, "boxIoUAt50": false }, { "attempt_id": "84c59fda4f2945ead7a21ab6", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 387.077, "cost": 0.00017052, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "720e81d83020c0dca936f21b", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 389.458, "cost": 0.00017052, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.6653760504857125, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1768dc421dbb74cf8d686941", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 357.2, "cost": 0.000160776, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e991e7e0693683679327cf43", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 873.879, "cost": 0.000160692, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "031e5a46a7891601512cfe7f", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 455.52, "cost": 0.00016296, "schemaValid": true, "pointHit": true, "boxIoU": 0.869019039849416, "boxIoUAt50": true }, { "attempt_id": "4a1898a8b7edf820c2aaa9ed", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 656.185, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.851017023045835, "boxIoUAt50": true }, { "attempt_id": "9e92c5d7c574a2738e75f30c", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 390.537, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.03564422112705726, "boxIoUAt50": false }, { "attempt_id": "716c9acde4eb176f83d2dcc8", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 351.095, "cost": 0.000163044, "schemaValid": true, "pointHit": true, "boxIoU": 0.61436172827071, "boxIoUAt50": true }, { "attempt_id": "fe29fc7ebdee539c6dc0f240", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 327.228, "cost": 0.000170394, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "324f00c307fbe2d3cbbd814a", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 366.38, "cost": 0.000170478, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.12765957990040766, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "79b970d06c50e19ac109a21e", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 334.677, "cost": 0.000160734, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d92cbe91831b7eaea94a14a9", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 384.814, "cost": 0.00016065, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "315abc35b8134e34989ddcfa", "case_id": "banking-ios/clean/grounding/profile", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 325.748, "cost": 0.000055272, "schemaValid": true, "pointHit": true, "boxIoU": 0.7406094704642425, "boxIoUAt50": true }, { "attempt_id": "28443b0a1bf0883ce344e591", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 303.02, "cost": 0.000055314, "schemaValid": true, "pointHit": true, "boxIoU": 0.6314275263134961, "boxIoUAt50": true }, { "attempt_id": "36a176e6556d278621d268a5", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 332.371, "cost": 0.000055272, "schemaValid": true, "pointHit": true, "boxIoU": 0.030718705711512147, "boxIoUAt50": false }, { "attempt_id": "d93e634897ad1b502f40846f", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 409.545, "cost": 0.000055188, "schemaValid": true, "pointHit": true, "boxIoU": 0.8068542892247389, "boxIoUAt50": true }, { "attempt_id": "43169a1cc6b59c28b9b81583", "case_id": "banking-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 298.507, "cost": 0.000062748, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1333a7171c8adbc6b69c3d9b", "case_id": "banking-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 441.347, "cost": 0.000062916, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.17883060587496785, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c094efafca40528ee1c8eebc", "case_id": "banking-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 308.33, "cost": 0.000053046, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "970bb7decc90a53c102d9c01", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 285.596, "cost": 0.000053004, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c7e346026912f9ad5635dc9a", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 296.152, "cost": 0.000055356, "schemaValid": true, "pointHit": true, "boxIoU": 0.9302078134475861, "boxIoUAt50": true }, { "attempt_id": "31276964d1663eddfd1067cc", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 294.28, "cost": 0.000055314, "schemaValid": true, "pointHit": true, "boxIoU": 0.7039772890411281, "boxIoUAt50": true }, { "attempt_id": "47625edbf6937d7ad580bb5f", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 270.673, "cost": 0.00005523, "schemaValid": true, "pointHit": true, "boxIoU": 0.4430782577485107, "boxIoUAt50": false }, { "attempt_id": "2a22a4fb20675ea8b8421308", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 307.643, "cost": 0.000055314, "schemaValid": true, "pointHit": true, "boxIoU": 0.5296178647252424, "boxIoUAt50": true }, { "attempt_id": "83bb42592adb8bd47290b29e", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 324.126, "cost": 0.000062874, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fd0d0af4ba1dc144d4627533", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 309.697, "cost": 0.000062832, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "36e144dc5f513f2e90fdebe5", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 287.102, "cost": 0.000053004, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bca4491356a812a347c92152", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 288.843, "cost": 0.000053004, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0f00880c8aa8d0dd75e8c38d", "case_id": "smart-home-android/clean/grounding/more", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 316.32, "cost": 0.000055272, "schemaValid": true, "pointHit": true, "boxIoU": 0.6583195293015489, "boxIoUAt50": true }, { "attempt_id": "49f75e9b14e5d7c9e24aae45", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 299.305, "cost": 0.00005523, "schemaValid": true, "pointHit": true, "boxIoU": 0.7408935363666377, "boxIoUAt50": true }, { "attempt_id": "fe3d2308310e8f3012490ad1", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 377.545, "cost": 0.00005523, "schemaValid": true, "pointHit": true, "boxIoU": 0.8415755762538395, "boxIoUAt50": true }, { "attempt_id": "0f3f11cf05de7133a46d7418", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 2, "track": "grounding", "status": "success", "wall_time_ms": 319.467, "cost": 0.00005523, "schemaValid": true, "pointHit": true, "boxIoU": 0.8042013113136398, "boxIoUAt50": true }, { "attempt_id": "0b0022f765231cb866bb4e9e", "case_id": "smart-home-android/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 286.142, "cost": 0.000062706, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1dab1b8b3c4813ee9c6a89d4", "case_id": "smart-home-android/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 298.314, "cost": 0.00006279, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.23646553351479624, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "03ecf3c5ad77ac7a51836ab2", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 320.096, "cost": 0.000053088, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "adb71f359ab683db3300969c", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 347.243, "cost": 0.000052962, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dd2da72c7901ffc82dcdb3a4", "case_id": "commerce-catalog-desktop/clean/grounding/search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 347.585, "cost": 0.000162834, "schemaValid": true, "pointHit": true, "boxIoU": 0.08281870393145319, "boxIoUAt50": false }, { "attempt_id": "2d51f0f7b8792bd48d58f39e", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 369.421, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.9114781207715029, "boxIoUAt50": true }, { "attempt_id": "a0d0e6c9aa71b84d8e0b69d6", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 395.371, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.6520571060405461, "boxIoUAt50": true }, { "attempt_id": "963070f4471486f3fd32c2b5", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 454.848, "cost": 0.00016296, "schemaValid": true, "pointHit": true, "boxIoU": 0.1122177553061915, "boxIoUAt50": false }, { "attempt_id": "d6846fbb63bdb4327fe275f5", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 373.069, "cost": 0.00017052, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4ab2d58dc77cb71b3db50288", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 375.384, "cost": 0.00017052, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "65bbfcdbe0ef67eac4df7407", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 393.671, "cost": 0.000160734, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0790bd0f003d0c387ecb3f21", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 428.899, "cost": 0.000160818, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "567ed12d19d0ace46f565ef4", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 333.242, "cost": 0.000162918, "schemaValid": true, "pointHit": true, "boxIoU": 0.5141854196665914, "boxIoUAt50": true }, { "attempt_id": "a5d7c3596657eed69b146e45", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 348.973, "cost": 0.000162876, "schemaValid": true, "pointHit": true, "boxIoU": 0.9097723538148144, "boxIoUAt50": true }, { "attempt_id": "6a5dfe561a7aa6524f97a979", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 357.548, "cost": 0.000162918, "schemaValid": true, "pointHit": true, "boxIoU": 0.7380562667739121, "boxIoUAt50": true }, { "attempt_id": "122fed96b54bd2dd40525cba", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 375.114, "cost": 0.00016296, "schemaValid": true, "pointHit": true, "boxIoU": 0.11066119725816245, "boxIoUAt50": false }, { "attempt_id": "b19321abef9f3c81c0b20957", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 346.219, "cost": 0.000170436, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3bb1c11b2a17c001152bca7d", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 357.881, "cost": 0.00017031, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.7450484407699638, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "79ce58f9f7ec7089baf95522", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 358.286, "cost": 0.000160734, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8eb3945670336627ee4d8f86", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 388.094, "cost": 0.000160734, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e73911ede09a2f7dc55fe36c", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 328.925, "cost": 0.00016296, "schemaValid": true, "pointHit": true, "boxIoU": 0.7330249680333333, "boxIoUAt50": true }, { "attempt_id": "72020124e2364c7dab904846", "case_id": "iot-control-desktop/clean/grounding/restart-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 315.361, "cost": 0.00016296, "schemaValid": true, "pointHit": true, "boxIoU": 0.851017023045835, "boxIoUAt50": true }, { "attempt_id": "6cb2f78b2ab02745f8137152", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 344.964, "cost": 0.000162834, "schemaValid": true, "pointHit": true, "boxIoU": 0.03564422112705726, "boxIoUAt50": false }, { "attempt_id": "8fa77291ce32deae69278031", "case_id": "iot-control-desktop/clean/grounding/pump-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 347.438, "cost": 0.000163128, "schemaValid": true, "pointHit": true, "boxIoU": 0.4914893826165678, "boxIoUAt50": false }, { "attempt_id": "59387b0187fa81f145909755", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 365.903, "cost": 0.00017052, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "04aee37438cf7efa2cfa104e", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 373.387, "cost": 0.000170436, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.11010638766410157, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f912c15d838e42d33c4898c3", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 364.682, "cost": 0.000160734, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e3525be46c5ad70f74112621", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 440.363, "cost": 0.000160818, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e60a4af67865035417a4f23a", "case_id": "banking-ios/clean/grounding/profile", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 286.608, "cost": 0.00005523, "schemaValid": true, "pointHit": true, "boxIoU": 0.7406094704642425, "boxIoUAt50": true }, { "attempt_id": "adfe8bcd8037e7f8d9d18add", "case_id": "banking-ios/clean/grounding/transaction-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 285.615, "cost": 0.000055146, "schemaValid": true, "pointHit": true, "boxIoU": 0.6249101412643956, "boxIoUAt50": true }, { "attempt_id": "c9cf414ab8ed781a03d29456", "case_id": "banking-ios/clean/grounding/hide-balance", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 285.217, "cost": 0.000055146, "schemaValid": true, "pointHit": true, "boxIoU": 0.030718705711512147, "boxIoUAt50": false }, { "attempt_id": "7dc13cdcff87264325dbcfba", "case_id": "banking-ios/clean/grounding/transfer", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 331.257, "cost": 0.000055104, "schemaValid": true, "pointHit": true, "boxIoU": 0.8068542892247389, "boxIoUAt50": true }, { "attempt_id": "e24d4c7de43ac0f63ed20fc1", "case_id": "banking-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 437.722, "cost": 0.000062622, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5026a3c0abaf743aa8a59cc9", "case_id": "banking-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 296.045, "cost": 0.000062748, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.17883060587496785, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3c419749da59b8fddd530783", "case_id": "banking-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 279.447, "cost": 0.000053046, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "58455809717e48adbdeafca2", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 444.19, "cost": 0.000052878, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4a01f11b75b4a6f3af76cde4", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 315.934, "cost": 0.000055398, "schemaValid": true, "pointHit": true, "boxIoU": 0.9405581814225531, "boxIoUAt50": true }, { "attempt_id": "ed09039f5a874da8b22406b5", "case_id": "flight-pass-ios/clean/grounding/check-in", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 297.937, "cost": 0.000055146, "schemaValid": true, "pointHit": true, "boxIoU": 0.7039772890411281, "boxIoUAt50": true }, { "attempt_id": "b908d6364c91ea52edadde02", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 293.382, "cost": 0.000055188, "schemaValid": true, "pointHit": true, "boxIoU": 0.41406360816683774, "boxIoUAt50": false }, { "attempt_id": "a6a5be153f8591560ccf651b", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 275.238, "cost": 0.00005523, "schemaValid": true, "pointHit": true, "boxIoU": 0.5296178647252424, "boxIoUAt50": true }, { "attempt_id": "8b32053be9ec6a7a873ee226", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 357.549, "cost": 0.000062706, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1176903b976c1d611a7ed140", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 303.617, "cost": 0.00006279, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": 0, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "974feef5b5155501935d2de8", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 286.551, "cost": 0.000053088, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1b0ee74f320d09d96674c09e", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 290.911, "cost": 0.000053088, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "47c8a52de32e634e13d8f601", "case_id": "smart-home-android/clean/grounding/more", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 293.316, "cost": 0.000055314, "schemaValid": true, "pointHit": true, "boxIoU": 0.4937396469761617, "boxIoUAt50": false }, { "attempt_id": "c0582f55d3468870783bcabe", "case_id": "smart-home-android/clean/grounding/room-search", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 391.987, "cost": 0.000055398, "schemaValid": true, "pointHit": true, "boxIoU": 0.7408935363666377, "boxIoUAt50": true }, { "attempt_id": "3822300ad266fd025c00f27f", "case_id": "smart-home-android/clean/grounding/thermostat-card", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 352.234, "cost": 0.000055188, "schemaValid": true, "pointHit": true, "boxIoU": 0.8415755762538395, "boxIoUAt50": true }, { "attempt_id": "e19ffffc9024f5c9c018cd16", "case_id": "smart-home-android/clean/grounding/add-device", "repetition": 3, "track": "grounding", "status": "success", "wall_time_ms": 346.539, "cost": 0.000055188, "schemaValid": true, "pointHit": true, "boxIoU": 0.8042013113136398, "boxIoUAt50": true }, { "attempt_id": "2819e52a559c6f65bc4c0ce7", "case_id": "smart-home-android/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 346.72, "cost": 0.000062748, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1d3fdfd1bfc8fbc03b391ec2", "case_id": "smart-home-android/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 531.481, "cost": 0.00006279, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": 0.23646553351479624, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6df9c15b42a6e1d2100f3172", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 390.6, "cost": 0.000053046, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "078291f35bdad3c088b97f84", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 338.546, "cost": 0.000053088, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] }, "openai/gpt-6-luna-decisions": { "name": "GPT-6 Luna Decisions", "classification_only": true, "point_source": null, "planned_requests": 72, "completed_requests": 72, "transport_attempts": 72, "request_failures": 0, "invalid_responses": 0, "resolved_models": [ "openai/gpt-6-luna-decisions-20261006" ], "grounding": null, "presence": { "metrics": { "balancedAccuracy": 1, "presentRecall": 1, "absentSpecificity": 1, "falsePositiveRate": 0, "schemaValidRate": 1 }, "correct": 36, "count": 36 }, "defects": { "metrics": { "balancedAccuracy": 0.8333333333333333, "recall": 0.6666666666666666, "specificity": 1, "falsePositiveRate": 0, "categoryAccuracy": 0.5, "meanLocalizationIoU": null, "schemaValidRate": 1 }, "correct": 30, "count": 36, "category_correct": 9, "category_count": 18, "region_supported": false }, "operations_by_track": { "grounding": { "count": 0, "transport_attempts": 0, "p50_ms": null, "p95_ms": null, "cost_usd": 0 }, "element-presence": { "count": 36, "transport_attempts": 36, "p50_ms": 305.62, "p95_ms": 893.257, "cost_usd": 0.0038235000000000005 }, "layout-defect": { "count": 36, "transport_attempts": 36, "p50_ms": 361.72249999999997, "p95_ms": 826.192, "cost_usd": 0.004647599999999999 } }, "operations": { "p50_ms": 338.0585, "p95_ms": 898.0498, "cost_usd": 0.008471100000000004, "missing_cost_count": 0 }, "attempts": [ { "attempt_id": "9ed6b9f0469f5227ca8ce4ba", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 375.38, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "834495a367ff9f02d78b82b8", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 454.438, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "54f1788bccbe94940c95186d", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 317.746, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ffe009f4e57e5b7386bb33c3", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 353.046, "cost": 0.000154, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4a65408a383103ef2e2c45f6", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 375.261, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ef4650c616dda638dc555130", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 361.578, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f49f3228773e63d9f9a7fdba", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 300.107, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "82714724714f99d6e2b2a585", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 382.693, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ef0b680b2207e06f07c4acd3", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 322.418, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a8eb98c4d844e86690178e74", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 519.935, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8b16abd5f31221d619b59688", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 550.585, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "863544f1df1281a2f201e674", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 258.807, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "192a996a0fbce8ff2ab38778", "case_id": "banking-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 237.759, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "004e64a9b35461a2ecca67a9", "case_id": "banking-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 210.789, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "e892d89aca784c078edb892d", "case_id": "banking-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 287.712, "cost": 0.0000584, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "86266392b3b19d4eb1279f61", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 220.127, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "91e9aa148f3b7c8843fbbf39", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 526.262, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7ced6d0c052e81d42710c000", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 662.647, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "27fab540127a1e8fd17fdf8f", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 380.498, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "40fb95b45277a4d807a157a4", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 193.4, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c0226619b18e6d4225646cb9", "case_id": "smart-home-android/clean/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 328.649, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "cffe533c82d3df94be6f2065", "case_id": "smart-home-android/defect/layout-defect", "repetition": 1, "track": "layout-defect", "status": "success", "wall_time_ms": 266.481, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d65fcc1715cd7cf9c7b6e15c", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 256.759, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "930c61e36fceff98a6a0f18a", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 1, "track": "element-presence", "status": "success", "wall_time_ms": 209.737, "cost": 0.0000586, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "13533fa4e34c415eed16053d", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 347.468, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "73b89f7fa29a60e7db4bacee", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 383.28, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "0543055d62f65375416d2b22", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 271.25, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7aa5d70e97ea06fd53218155", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 356.842, "cost": 0.000154, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "1ef61237f311a53a18eee92e", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 349.478, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "576ff4fc410f8bc2dd2ebf7f", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 321.454, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "f1e6582be4d1cb973149ead0", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 466.075, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "b73c2f09b48514409e8c7082", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 275.512, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4165a2bf8a08feffc94be054", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 737.988, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "28ed105b3e74e8e90dca7f60", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 600.704, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "fb4ac11200002cfcbfb43988", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 911.23, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "52e32f6e5555f11abda4bd0e", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 406.564, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7c01be0a62700d009ae39fab", "case_id": "banking-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 235.82, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "974c3306bdf66297fa592875", "case_id": "banking-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 208.93, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "27daefb6a3f30bed44626dca", "case_id": "banking-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 490.367, "cost": 0.0000584, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "68f86fedaeaabae37bbce190", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 235.109, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ef423bb4c8dba5e3338b87b7", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 615.117, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "dd1e2e52d9e06be93a18dfa0", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 266.266, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "7204e03f058e410cb0dcb1a1", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 260.042, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2430811f03b8a35534f75e13", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 228.523, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "63750643123cd3d84ba5ba59", "case_id": "smart-home-android/clean/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 236.025, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "911a3fe853814392012a6028", "case_id": "smart-home-android/defect/layout-defect", "repetition": 2, "track": "layout-defect", "status": "success", "wall_time_ms": 531.699, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "9a7cc5e9b0f8e41f613ad4e4", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 291.856, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a4044472cb2be82beafb9309", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 2, "track": "element-presence", "status": "success", "wall_time_ms": 194.827, "cost": 0.0000586, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c1c2c56101e9800ac36c4d11", "case_id": "commerce-catalog-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 489.699, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5918771bf30e582de772e630", "case_id": "commerce-catalog-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 246.745, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "9693a86000e2732602b532bf", "case_id": "commerce-catalog-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 497.037, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "8a0191eb916b73448c05e47b", "case_id": "commerce-catalog-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 481.258, "cost": 0.000154, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d633f044c9cc8d4162be078e", "case_id": "analytics-dashboard-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 363.463, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0444eee4fe2cb858d120af21", "case_id": "analytics-dashboard-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 231.298, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c55459d2b230db1bf84b2a0a", "case_id": "analytics-dashboard-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 1323.921, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "5f332ee1ec8be8cad28ddd49", "case_id": "analytics-dashboard-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 523.674, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "568a2f64e0ef27660c1634ea", "case_id": "iot-control-desktop/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 361.867, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "bea7af28fb49cb5c6e71c4e7", "case_id": "iot-control-desktop/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 366.684, "cost": 0.0001768, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "0ecf5bb26e3cf33ccc1a55ed", "case_id": "iot-control-desktop/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 311.133, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c4470afbe64d5633eb762bef", "case_id": "iot-control-desktop/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 277.312, "cost": 0.0001539, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "6a72b1b90044007b60966502", "case_id": "banking-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 246.25, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "27d97aa59b6792b93bb0f933", "case_id": "banking-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 252.52, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": false, "categoryCorrect": false, "localizationIoU": null, "truePositive": false, "trueNegative": false, "falsePositive": false, "falseNegative": true }, { "attempt_id": "78db7b2ef16d798ed3a3ee40", "case_id": "banking-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 269.787, "cost": 0.0000584, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "eed8f0604df41874368ea7cc", "case_id": "banking-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 522.823, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "a45e8d0eecce88306901c45a", "case_id": "flight-pass-ios/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 538.811, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "c43503798825ac5705be5e1a", "case_id": "flight-pass-ios/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1220.539, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": false, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "4ebdd9d7cbfc8fa2047c31ce", "case_id": "flight-pass-ios/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 193.866, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "e87cbe1cd0ede7280d7cc02e", "case_id": "flight-pass-ios/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 887.266, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "2b88c89ccec06132cace0263", "case_id": "smart-home-android/clean/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 1090.804, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": null, "localizationIoU": null, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false }, { "attempt_id": "3358ba49f94eb66049125de5", "case_id": "smart-home-android/defect/layout-defect", "repetition": 3, "track": "layout-defect", "status": "success", "wall_time_ms": 253.513, "cost": 0.0000814, "schemaValid": true, "classificationCorrect": true, "categoryCorrect": true, "localizationIoU": null, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "d627c8c82183bfb43403027c", "case_id": "smart-home-android/clean/element-presence/present", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 209.696, "cost": 0.0000585, "schemaValid": true, "classificationCorrect": true, "expectedFound": true, "predictedFound": true, "truePositive": true, "trueNegative": false, "falsePositive": false, "falseNegative": false }, { "attempt_id": "ec52eb74d4c18684c9dbad70", "case_id": "smart-home-android/clean/element-presence/absent", "repetition": 3, "track": "element-presence", "status": "success", "wall_time_ms": 316.139, "cost": 0.0000586, "schemaValid": true, "classificationCorrect": true, "expectedFound": false, "predictedFound": false, "truePositive": false, "trueNegative": true, "falsePositive": false, "falseNegative": false } ] } } }