{ "benchmark_version": "1.0.0", "edition": "2026-08-13", "scope": "smoke", "case_ids": [ "analytics-dashboard-desktop/clean/grounding/audience-tab", "analytics-dashboard-desktop/clean/grounding/compare-toggle", "analytics-dashboard-desktop/clean/grounding/revenue-card", "analytics-dashboard-desktop/clean/grounding/search-field", "banking-ios/clean/grounding/hide-balance", "banking-ios/clean/grounding/profile", "banking-ios/clean/grounding/transaction-search", "banking-ios/clean/grounding/transfer", "commerce-catalog-desktop/clean/grounding/add-product", "commerce-catalog-desktop/clean/grounding/catalog-query", "commerce-catalog-desktop/clean/grounding/search", "commerce-catalog-desktop/clean/grounding/stock-filter", "flight-pass-ios/clean/grounding/boarding-pass", "flight-pass-ios/clean/grounding/check-in", "flight-pass-ios/clean/grounding/profile-tab", "flight-pass-ios/clean/grounding/trips-tab", "iot-control-desktop/clean/grounding/auto-mode", "iot-control-desktop/clean/grounding/devices-nav", "iot-control-desktop/clean/grounding/pump-card", "iot-control-desktop/clean/grounding/restart-device", "smart-home-android/clean/grounding/add-device", "smart-home-android/clean/grounding/more", "smart-home-android/clean/grounding/room-search", "smart-home-android/clean/grounding/thermostat-card" ], "attempt_count": 216, "transport_attempt_count": 216, "response_count": 216, "request_failure_count": 0, "request_failure_rate": 0, "total_cost": 0.9327861500000001, "generation_metadata_missing": 98, "models": { "holo3-122b-a10b": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "metrics": { "pointAccuracy": 0.9166666666666666, "meanIoU": 0.4777360811322533, "iouAt50": 0.4166666666666667, "schemaValidRate": 0.9166666666666666 }, "repeat_consistency": null, "wall_time_ms": [ 3557.67, 1731.868, 1694.776, 1649.076, 1656.692, 1695.285, 1763.924, 1630.838, 2507.468, 1826.347, 1647.909, 1646.195, 1436.399, 1467.507, 1415.653, 1381.708, 1431.488, 1365.315, 1405.571, 1462.266, 1473.825, 1471.942, 1462.039, 1464.119 ], "total_cost": 0.012586999999999997, "attempts": [ { "attempt_id": "346819ae6be2662bf6ec0d20", "model": "holo3-122b-a10b", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "status": "success", "wall_time_ms": 3557.67, "cost": 0.0007118000000000001, "schemaValid": true, "pointHit": true, "boxIoU": 0.756868597613915, "boxIoUAt50": true }, { "attempt_id": "ee87ae3a86295084d81ad224", "model": "holo3-122b-a10b", "case_id": "commerce-catalog-desktop/clean/grounding/search", "status": "success", "wall_time_ms": 1731.868, "cost": 0.0006998, "schemaValid": true, "pointHit": true, "boxIoU": 0.5717189363602188, "boxIoUAt50": true }, { "attempt_id": "0786418cda8672e0e4eb5d88", "model": "holo3-122b-a10b", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "status": "success", "wall_time_ms": 1694.776, "cost": 0.0007238, "schemaValid": true, "pointHit": true, "boxIoU": 0.46277175281886207, "boxIoUAt50": false }, { "attempt_id": "62cb8a0d068e6810aa56dc06", "model": "holo3-122b-a10b", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "status": "success", "wall_time_ms": 1649.076, "cost": 0.0007062, "schemaValid": true, "pointHit": true, "boxIoU": 0.1251959101355944, "boxIoUAt50": false }, { "attempt_id": "0c8bb1df63e101255b7c29cb", "model": "holo3-122b-a10b", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "status": "success", "wall_time_ms": 1656.692, "cost": 0.0007144, "schemaValid": true, "pointHit": true, "boxIoU": 0.445630820309752, "boxIoUAt50": false }, { "attempt_id": "a14e06b1787f5683d71facda", "model": "holo3-122b-a10b", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "status": "invalid_response", "wall_time_ms": 1695.285, "cost": 0.0007238, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "e64281fcb587d10a9efef9fe", "model": "holo3-122b-a10b", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "status": "success", "wall_time_ms": 1763.924, "cost": 0.0007174, "schemaValid": true, "pointHit": true, "boxIoU": 0.41193119278192725, "boxIoUAt50": false }, { "attempt_id": "6b7ff4abfb8b3af277fd2e97", "model": "holo3-122b-a10b", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "status": "success", "wall_time_ms": 1630.838, "cost": 0.0006998, "schemaValid": true, "pointHit": true, "boxIoU": 0.3860219206014513, "boxIoUAt50": false }, { "attempt_id": "62824a7169c5eb352d2bca7a", "model": "holo3-122b-a10b", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "status": "success", "wall_time_ms": 2507.468, "cost": 0.0007238, "schemaValid": true, "pointHit": true, "boxIoU": 0.2155519449672533, "boxIoUAt50": false }, { "attempt_id": "7eef292259be1e214461c14c", "model": "holo3-122b-a10b", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "status": "success", "wall_time_ms": 1826.347, "cost": 0.0007178, "schemaValid": true, "pointHit": true, "boxIoU": 0.7439041939597648, "boxIoUAt50": true }, { "attempt_id": "a35db74ecf4e0a289985882a", "model": "holo3-122b-a10b", "case_id": "iot-control-desktop/clean/grounding/pump-card", "status": "invalid_response", "wall_time_ms": 1647.909, "cost": 0.000689, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "07e81237999dbc2c346a0e0d", "model": "holo3-122b-a10b", "case_id": "iot-control-desktop/clean/grounding/restart-device", "status": "success", "wall_time_ms": 1646.195, "cost": 0.0007118000000000001, "schemaValid": true, "pointHit": true, "boxIoU": 0.8386952689118515, "boxIoUAt50": true }, { "attempt_id": "6b7c59faf0c42ae44c374f3a", "model": "holo3-122b-a10b", "case_id": "banking-ios/clean/grounding/transfer", "status": "success", "wall_time_ms": 1436.399, "cost": 0.0003412, "schemaValid": true, "pointHit": true, "boxIoU": 0.8814409804873702, "boxIoUAt50": true }, { "attempt_id": "d024034fb8573ed0a1c25020", "model": "holo3-122b-a10b", "case_id": "banking-ios/clean/grounding/profile", "status": "success", "wall_time_ms": 1467.507, "cost": 0.00035059999999999996, "schemaValid": true, "pointHit": true, "boxIoU": 0.45948204689435374, "boxIoUAt50": false }, { "attempt_id": "ba3e8e28c014fd39951d570e", "model": "holo3-122b-a10b", "case_id": "banking-ios/clean/grounding/transaction-search", "status": "success", "wall_time_ms": 1415.653, "cost": 0.0003236, "schemaValid": true, "pointHit": true, "boxIoU": 0.25744700278500826, "boxIoUAt50": false }, { "attempt_id": "0d10e225da5d67e6e6c2d2ae", "model": "holo3-122b-a10b", "case_id": "banking-ios/clean/grounding/hide-balance", "status": "success", "wall_time_ms": 1381.708, "cost": 0.0003116, "schemaValid": true, "pointHit": true, "boxIoU": 0.18092719465404503, "boxIoUAt50": false }, { "attempt_id": "9e792dcdde78fce63fd9b4ab", "model": "holo3-122b-a10b", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "status": "success", "wall_time_ms": 1431.488, "cost": 0.00034500000000000004, "schemaValid": true, "pointHit": true, "boxIoU": 0.2668657263484307, "boxIoUAt50": false }, { "attempt_id": "766148fdc4ffd9ee0d049f98", "model": "holo3-122b-a10b", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "status": "success", "wall_time_ms": 1365.315, "cost": 0.0003132, "schemaValid": true, "pointHit": true, "boxIoU": 0.5766986809838064, "boxIoUAt50": true }, { "attempt_id": "9aa36fd281a385ae50b38d02", "model": "holo3-122b-a10b", "case_id": "flight-pass-ios/clean/grounding/check-in", "status": "success", "wall_time_ms": 1405.571, "cost": 0.0003416, "schemaValid": true, "pointHit": true, "boxIoU": 0.8710692506084212, "boxIoUAt50": true }, { "attempt_id": "746b78604d033b718c276e4c", "model": "holo3-122b-a10b", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "status": "success", "wall_time_ms": 1462.266, "cost": 0.0003746, "schemaValid": true, "pointHit": true, "boxIoU": 0.7371984984870238, "boxIoUAt50": true }, { "attempt_id": "45e4b9fae9008dc498383237", "model": "holo3-122b-a10b", "case_id": "smart-home-android/clean/grounding/thermostat-card", "status": "success", "wall_time_ms": 1473.825, "cost": 0.000333, "schemaValid": true, "pointHit": true, "boxIoU": 0.6872297996426673, "boxIoUAt50": true }, { "attempt_id": "181383b3668f943945eccc89", "model": "holo3-122b-a10b", "case_id": "smart-home-android/clean/grounding/add-device", "status": "success", "wall_time_ms": 1471.942, "cost": 0.0003416, "schemaValid": true, "pointHit": true, "boxIoU": 0.8805744655593507, "boxIoUAt50": true }, { "attempt_id": "60a1cfec303e707df7a4a67a", "model": "holo3-122b-a10b", "case_id": "smart-home-android/clean/grounding/more", "status": "success", "wall_time_ms": 1462.039, "cost": 0.00032700000000000003, "schemaValid": true, "pointHit": true, "boxIoU": 0.46097498587639313, "boxIoUAt50": false }, { "attempt_id": "f7b3014a5722c7fddda0625c", "model": "holo3-122b-a10b", "case_id": "smart-home-android/clean/grounding/room-search", "status": "success", "wall_time_ms": 1464.119, "cost": 0.00034460000000000003, "schemaValid": true, "pointHit": true, "boxIoU": 0.24746677638662093, "boxIoUAt50": false } ] }, "x-ai/grok-4.5": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "metrics": { "pointAccuracy": 0.8333333333333334, "meanIoU": 0.44456354634343537, "iouAt50": 0.5, "schemaValidRate": 1 }, "repeat_consistency": null, "wall_time_ms": [ 1963.45, 1452.915, 1600.032, 1697.83, 1780.881, 1520.876, 1918.96, 1597.804, 1608.753, 1626.407, 1711.242, 1659.441, 1727.441, 1412.199, 2039.581, 1614.264, 1707.405, 1551.135, 2068.73, 2051.29, 1480.53, 1526.208, 1308.888, 1386.324 ], "total_cost": 0.06688759999999996, "attempts": [ { "attempt_id": "f230db32efb09258105586bf", "model": "x-ai/grok-4.5", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "status": "success", "wall_time_ms": 1963.45, "cost": 0.0037884, "schemaValid": true, "pointHit": true, "boxIoU": 0.5455196803103946, "boxIoUAt50": true }, { "attempt_id": "e35559755189cc8781f151cf", "model": "x-ai/grok-4.5", "case_id": "commerce-catalog-desktop/clean/grounding/search", "status": "success", "wall_time_ms": 1452.915, "cost": 0.0037884, "schemaValid": true, "pointHit": true, "boxIoU": 0.3386175970161495, "boxIoUAt50": false }, { "attempt_id": "38ae196012c93cd1b50748fc", "model": "x-ai/grok-4.5", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "status": "success", "wall_time_ms": 1600.032, "cost": 0.0037884, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "fa787f4a1fb4379ab224e779", "model": "x-ai/grok-4.5", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "status": "success", "wall_time_ms": 1697.83, "cost": 0.0037904, "schemaValid": true, "pointHit": false, "boxIoU": 0.04339176714214022, "boxIoUAt50": false }, { "attempt_id": "a4baf12cff7cbb01f620afaa", "model": "x-ai/grok-4.5", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "status": "success", "wall_time_ms": 1780.881, "cost": 0.0037864, "schemaValid": true, "pointHit": true, "boxIoU": 0.43979883386727664, "boxIoUAt50": false }, { "attempt_id": "42d25c205a15a88dc46b7155", "model": "x-ai/grok-4.5", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "status": "success", "wall_time_ms": 1520.876, "cost": 0.0037884, "schemaValid": true, "pointHit": true, "boxIoU": 0.2589259066314708, "boxIoUAt50": false }, { "attempt_id": "0d3c2a8aa7eeefe0da86008c", "model": "x-ai/grok-4.5", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "status": "success", "wall_time_ms": 1918.96, "cost": 0.0037864, "schemaValid": true, "pointHit": true, "boxIoU": 0.6669149476135369, "boxIoUAt50": true }, { "attempt_id": "51c4389730286ed80b786171", "model": "x-ai/grok-4.5", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "status": "success", "wall_time_ms": 1597.804, "cost": 0.0037884, "schemaValid": true, "pointHit": true, "boxIoU": 0.7283013630285208, "boxIoUAt50": true }, { "attempt_id": "6cc756c9fa96aa79648263ca", "model": "x-ai/grok-4.5", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "status": "success", "wall_time_ms": 1608.753, "cost": 0.0037884, "schemaValid": true, "pointHit": true, "boxIoU": 0.3924879787859363, "boxIoUAt50": false }, { "attempt_id": "9730b393ea710f9e5018ba5f", "model": "x-ai/grok-4.5", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "status": "success", "wall_time_ms": 1626.407, "cost": 0.0037884, "schemaValid": true, "pointHit": true, "boxIoU": 0.6016683206471826, "boxIoUAt50": true }, { "attempt_id": "5bea7b9422d6cb66cad3bee4", "model": "x-ai/grok-4.5", "case_id": "iot-control-desktop/clean/grounding/pump-card", "status": "success", "wall_time_ms": 1711.242, "cost": 0.0037944, "schemaValid": true, "pointHit": false, "boxIoU": 0.1465230967791025, "boxIoUAt50": false }, { "attempt_id": "29e59b1b362a73a8c8117b45", "model": "x-ai/grok-4.5", "case_id": "iot-control-desktop/clean/grounding/restart-device", "status": "success", "wall_time_ms": 1659.441, "cost": 0.0037884, "schemaValid": true, "pointHit": true, "boxIoU": 0.24630060331961642, "boxIoUAt50": false }, { "attempt_id": "a502f0603483b1624c6a6887", "model": "x-ai/grok-4.5", "case_id": "banking-ios/clean/grounding/transfer", "status": "success", "wall_time_ms": 1727.441, "cost": 0.0018784, "schemaValid": true, "pointHit": true, "boxIoU": 0.625146378746224, "boxIoUAt50": true }, { "attempt_id": "f53f6b4dc5da78aa5576c38e", "model": "x-ai/grok-4.5", "case_id": "banking-ios/clean/grounding/profile", "status": "success", "wall_time_ms": 1412.199, "cost": 0.0018804, "schemaValid": true, "pointHit": true, "boxIoU": 0.5280775379451728, "boxIoUAt50": true }, { "attempt_id": "8548dfd4d04f7656d2a69de9", "model": "x-ai/grok-4.5", "case_id": "banking-ios/clean/grounding/transaction-search", "status": "success", "wall_time_ms": 2039.581, "cost": 0.0018804, "schemaValid": true, "pointHit": true, "boxIoU": 0.34959601188324235, "boxIoUAt50": false }, { "attempt_id": "f57f94dd55d746ff8caaec73", "model": "x-ai/grok-4.5", "case_id": "banking-ios/clean/grounding/hide-balance", "status": "success", "wall_time_ms": 1614.264, "cost": 0.0018804, "schemaValid": true, "pointHit": false, "boxIoU": 0.02866843493163971, "boxIoUAt50": false }, { "attempt_id": "98a38587d59f1ebd8ced491b", "model": "x-ai/grok-4.5", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "status": "success", "wall_time_ms": 1707.405, "cost": 0.0018824, "schemaValid": true, "pointHit": true, "boxIoU": 0.38626538952014466, "boxIoUAt50": false }, { "attempt_id": "506f0add290635c2b55c03b3", "model": "x-ai/grok-4.5", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "status": "success", "wall_time_ms": 1551.135, "cost": 0.0018884, "schemaValid": true, "pointHit": true, "boxIoU": 0.6986652936945741, "boxIoUAt50": true }, { "attempt_id": "03573fed086dd84259266bd2", "model": "x-ai/grok-4.5", "case_id": "flight-pass-ios/clean/grounding/check-in", "status": "success", "wall_time_ms": 2068.73, "cost": 0.0018804, "schemaValid": true, "pointHit": true, "boxIoU": 0.6313531552908186, "boxIoUAt50": true }, { "attempt_id": "9a2b546f9f1181adf82427de", "model": "x-ai/grok-4.5", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "status": "success", "wall_time_ms": 2051.29, "cost": 0.0018804, "schemaValid": true, "pointHit": true, "boxIoU": 0.5031808771528256, "boxIoUAt50": true }, { "attempt_id": "673d1ab08db35b4105905f37", "model": "x-ai/grok-4.5", "case_id": "smart-home-android/clean/grounding/thermostat-card", "status": "success", "wall_time_ms": 1480.53, "cost": 0.0021, "schemaValid": true, "pointHit": true, "boxIoU": 0.794728170516944, "boxIoUAt50": true }, { "attempt_id": "c6569757921651debd711b65", "model": "x-ai/grok-4.5", "case_id": "smart-home-android/clean/grounding/add-device", "status": "success", "wall_time_ms": 1526.208, "cost": 0.0018144, "schemaValid": true, "pointHit": true, "boxIoU": 0.6535423541103411, "boxIoUAt50": true }, { "attempt_id": "b802b37906a76300d8aaa4b8", "model": "x-ai/grok-4.5", "case_id": "smart-home-android/clean/grounding/more", "status": "success", "wall_time_ms": 1308.888, "cost": 0.0012296, "schemaValid": true, "pointHit": true, "boxIoU": 0.4433254910254152, "boxIoUAt50": false }, { "attempt_id": "51f5fc2af83bf2c21831be0b", "model": "x-ai/grok-4.5", "case_id": "smart-home-android/clean/grounding/room-search", "status": "success", "wall_time_ms": 1386.324, "cost": 0.0012276, "schemaValid": true, "pointHit": true, "boxIoU": 0.6185259222837804, "boxIoUAt50": true } ] }, "openai/gpt-5.4": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "metrics": { "pointAccuracy": 0.75, "meanIoU": 0.5282906619204312, "iouAt50": 0.5416666666666666, "schemaValidRate": 1 }, "repeat_consistency": null, "wall_time_ms": [ 3289.199, 2161.831, 2847.882, 5362.731, 3048.526, 3762.725, 3021.821, 5929.199, 3066.607, 3621.069, 2890.509, 2332.406, 3089.439, 2260.731, 1602.55, 6706.505, 2820.549, 1541.276, 1652.631, 3292.143, 1977.654, 1502.166, 1912.834, 2069.692 ], "total_cost": 0.119625, "attempts": [ { "attempt_id": "7b3bcfc266948dee1c7d4e29", "model": "openai/gpt-5.4", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "status": "success", "wall_time_ms": 3289.199, "cost": 0.006605, "schemaValid": true, "pointHit": true, "boxIoU": 0.9734577622286735, "boxIoUAt50": true }, { "attempt_id": "d0166371a95423ed4e8bd92a", "model": "openai/gpt-5.4", "case_id": "commerce-catalog-desktop/clean/grounding/search", "status": "success", "wall_time_ms": 2161.831, "cost": 0.006035, "schemaValid": true, "pointHit": true, "boxIoU": 0.9031302572728028, "boxIoUAt50": true }, { "attempt_id": "d8b3d204d191f5c23aa9cf37", "model": "openai/gpt-5.4", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "status": "success", "wall_time_ms": 2847.882, "cost": 0.006575, "schemaValid": true, "pointHit": true, "boxIoU": 0.823359576632034, "boxIoUAt50": true }, { "attempt_id": "9b65fb110e8ce0476e9cdb20", "model": "openai/gpt-5.4", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "status": "success", "wall_time_ms": 5362.731, "cost": 0.0076725, "schemaValid": true, "pointHit": true, "boxIoU": 0.5136897173780947, "boxIoUAt50": true }, { "attempt_id": "99dadb90a4af52e055838d48", "model": "openai/gpt-5.4", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "status": "success", "wall_time_ms": 3048.526, "cost": 0.0070225, "schemaValid": true, "pointHit": true, "boxIoU": 0.9558804507543094, "boxIoUAt50": true }, { "attempt_id": "63a976a2dd7cdccefce08424", "model": "openai/gpt-5.4", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "status": "success", "wall_time_ms": 3762.725, "cost": 0.007535, "schemaValid": true, "pointHit": true, "boxIoU": 0.5300402708707087, "boxIoUAt50": true }, { "attempt_id": "758e35c4b64b72fe942bc4a5", "model": "openai/gpt-5.4", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "status": "success", "wall_time_ms": 3021.821, "cost": 0.0066775, "schemaValid": true, "pointHit": true, "boxIoU": 0.3166916097901603, "boxIoUAt50": false }, { "attempt_id": "e9e12042dc1e63014034f7ef", "model": "openai/gpt-5.4", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "status": "success", "wall_time_ms": 5929.199, "cost": 0.007355, "schemaValid": true, "pointHit": true, "boxIoU": 0.9906016731066115, "boxIoUAt50": true }, { "attempt_id": "223a255d69fac5071612f579", "model": "openai/gpt-5.4", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "status": "success", "wall_time_ms": 3066.607, "cost": 0.00647, "schemaValid": true, "pointHit": true, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "65c4175fd4da5b1ba4c5be4b", "model": "openai/gpt-5.4", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "status": "success", "wall_time_ms": 3621.069, "cost": 0.006065, "schemaValid": true, "pointHit": true, "boxIoU": 0.946914617936731, "boxIoUAt50": true }, { "attempt_id": "14b91b2d0bd2cbaaf2e6ede8", "model": "openai/gpt-5.4", "case_id": "iot-control-desktop/clean/grounding/pump-card", "status": "success", "wall_time_ms": 2890.509, "cost": 0.0061475, "schemaValid": true, "pointHit": false, "boxIoU": 0.14586348844963687, "boxIoUAt50": false }, { "attempt_id": "1efc068bb1f051d512c6f32f", "model": "openai/gpt-5.4", "case_id": "iot-control-desktop/clean/grounding/restart-device", "status": "success", "wall_time_ms": 2332.406, "cost": 0.00614, "schemaValid": true, "pointHit": true, "boxIoU": 0.963489970529028, "boxIoUAt50": true }, { "attempt_id": "069f00b49859e982de00d565", "model": "openai/gpt-5.4", "case_id": "banking-ios/clean/grounding/transfer", "status": "success", "wall_time_ms": 3089.439, "cost": 0.003815, "schemaValid": true, "pointHit": true, "boxIoU": 0.9667670396505261, "boxIoUAt50": true }, { "attempt_id": "198201744dccb904b6957017", "model": "openai/gpt-5.4", "case_id": "banking-ios/clean/grounding/profile", "status": "success", "wall_time_ms": 2260.731, "cost": 0.0033525, "schemaValid": true, "pointHit": true, "boxIoU": 0.9709006277987463, "boxIoUAt50": true }, { "attempt_id": "e6fb28ad13e159131994ab3c", "model": "openai/gpt-5.4", "case_id": "banking-ios/clean/grounding/transaction-search", "status": "success", "wall_time_ms": 1602.55, "cost": 0.0026475, "schemaValid": true, "pointHit": true, "boxIoU": 0.11965511006375433, "boxIoUAt50": false }, { "attempt_id": "6d765066b28f507b2d5de746", "model": "openai/gpt-5.4", "case_id": "banking-ios/clean/grounding/hide-balance", "status": "success", "wall_time_ms": 6706.505, "cost": 0.0050775, "schemaValid": true, "pointHit": true, "boxIoU": 0.2874142426663367, "boxIoUAt50": false }, { "attempt_id": "ac02b69a0dd900f37279cc68", "model": "openai/gpt-5.4", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "status": "success", "wall_time_ms": 2820.549, "cost": 0.0040125, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "dd1fbb40215e91108e5eaa9c", "model": "openai/gpt-5.4", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "status": "success", "wall_time_ms": 1541.276, "cost": 0.002895, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "7216f4f725f7be19c35db7d8", "model": "openai/gpt-5.4", "case_id": "flight-pass-ios/clean/grounding/check-in", "status": "success", "wall_time_ms": 1652.631, "cost": 0.0027375, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "b3b11de85a20604ff071f3f7", "model": "openai/gpt-5.4", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "status": "success", "wall_time_ms": 3292.143, "cost": 0.0037425, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "08b5663e025392e94779b4ee", "model": "openai/gpt-5.4", "case_id": "smart-home-android/clean/grounding/thermostat-card", "status": "success", "wall_time_ms": 1977.654, "cost": 0.003115, "schemaValid": true, "pointHit": true, "boxIoU": 0.9433760778153185, "boxIoUAt50": true }, { "attempt_id": "ffbce070b02f1d5b1baed1e2", "model": "openai/gpt-5.4", "case_id": "smart-home-android/clean/grounding/add-device", "status": "success", "wall_time_ms": 1502.166, "cost": 0.0024825, "schemaValid": true, "pointHit": true, "boxIoU": 0.9193908283463966, "boxIoUAt50": true }, { "attempt_id": "1d7859b56424aee63f4e23e4", "model": "openai/gpt-5.4", "case_id": "smart-home-android/clean/grounding/more", "status": "success", "wall_time_ms": 1912.834, "cost": 0.002965, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "4a76f5c9d8bd4a7c7ed5ff72", "model": "openai/gpt-5.4", "case_id": "smart-home-android/clean/grounding/room-search", "status": "success", "wall_time_ms": 2069.692, "cost": 0.0024825, "schemaValid": true, "pointHit": true, "boxIoU": 0.40835256480048, "boxIoUAt50": false } ] }, "google/gemini-3.6-flash": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "metrics": { "pointAccuracy": 0.7916666666666666, "meanIoU": 0.6451082740899948, "iouAt50": 0.6666666666666666, "schemaValidRate": 0.7916666666666666 }, "repeat_consistency": null, "wall_time_ms": [ 1893.326, 1358.619, 1387.424, 1366.521, 1673.867, 3313.648, 1209.485, 2915.123, 3790.818, 1348.014, 1198.304, 1328.077, 1888.517, 1535.132, 1125.072, 1563.019, 3485.663, 1293.231, 1198.286, 3296.711, 1558.857, 1773.689, 1344.212, 1192.967 ], "total_cost": 0.06372150000000001, "attempts": [ { "attempt_id": "43e7ed3669df0bd3a72d1e18", "model": "google/gemini-3.6-flash", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "status": "success", "wall_time_ms": 1893.326, "cost": 0.002202, "schemaValid": true, "pointHit": true, "boxIoU": 0.9932670824297664, "boxIoUAt50": true }, { "attempt_id": "2897f4932528660c4d7d595b", "model": "google/gemini-3.6-flash", "case_id": "commerce-catalog-desktop/clean/grounding/search", "status": "success", "wall_time_ms": 1358.619, "cost": 0.002082, "schemaValid": true, "pointHit": true, "boxIoU": 0.8785808812979015, "boxIoUAt50": true }, { "attempt_id": "0ba4fa57a05d90fdf6bb2867", "model": "google/gemini-3.6-flash", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "status": "success", "wall_time_ms": 1387.424, "cost": 0.0021045, "schemaValid": true, "pointHit": true, "boxIoU": 0.8425648661053785, "boxIoUAt50": true }, { "attempt_id": "c9ba413fd1c151de0a5f158c", "model": "google/gemini-3.6-flash", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "status": "success", "wall_time_ms": 1366.521, "cost": 0.0020985, "schemaValid": true, "pointHit": true, "boxIoU": 0.0805840714870343, "boxIoUAt50": false }, { "attempt_id": "95992cafbf4bac113b08aefe", "model": "google/gemini-3.6-flash", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "status": "success", "wall_time_ms": 1673.867, "cost": 0.0022005, "schemaValid": true, "pointHit": true, "boxIoU": 0.9558804507543094, "boxIoUAt50": true }, { "attempt_id": "6b49115e6e07fe309ab010cd", "model": "google/gemini-3.6-flash", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "status": "invalid_response", "wall_time_ms": 3313.648, "cost": 0.0046845, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "0a786fe638a058cc87ca7fea", "model": "google/gemini-3.6-flash", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "status": "success", "wall_time_ms": 1209.485, "cost": 0.0021855, "schemaValid": true, "pointHit": true, "boxIoU": 0.7700279687274331, "boxIoUAt50": true }, { "attempt_id": "c8c2911b9478d7632fc4c955", "model": "google/gemini-3.6-flash", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "status": "invalid_response", "wall_time_ms": 2915.123, "cost": 0.004692, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "4c64e26ea027b8670f97727d", "model": "google/gemini-3.6-flash", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "status": "invalid_response", "wall_time_ms": 3790.818, "cost": 0.004692, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "0f927eaa213f3860e0a79549", "model": "google/gemini-3.6-flash", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "status": "success", "wall_time_ms": 1348.014, "cost": 0.002187, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "87d77c9cfc309bb82d0566ab", "model": "google/gemini-3.6-flash", "case_id": "iot-control-desktop/clean/grounding/pump-card", "status": "success", "wall_time_ms": 1198.304, "cost": 0.002109, "schemaValid": true, "pointHit": true, "boxIoU": 0.9759825860517736, "boxIoUAt50": true }, { "attempt_id": "fc5c4665ed5a98f34bfa5bf9", "model": "google/gemini-3.6-flash", "case_id": "iot-control-desktop/clean/grounding/restart-device", "status": "success", "wall_time_ms": 1328.077, "cost": 0.0021045, "schemaValid": true, "pointHit": true, "boxIoU": 0.9762846034133694, "boxIoUAt50": true }, { "attempt_id": "72c5deed0e050f09834b9760", "model": "google/gemini-3.6-flash", "case_id": "banking-ios/clean/grounding/transfer", "status": "success", "wall_time_ms": 1888.517, "cost": 0.0020745, "schemaValid": true, "pointHit": true, "boxIoU": 0.9605228602797484, "boxIoUAt50": true }, { "attempt_id": "3535daef65116b7b8493b262", "model": "google/gemini-3.6-flash", "case_id": "banking-ios/clean/grounding/profile", "status": "success", "wall_time_ms": 1535.132, "cost": 0.002166, "schemaValid": true, "pointHit": true, "boxIoU": 0.9392126771521667, "boxIoUAt50": true }, { "attempt_id": "1913f3f477cdf76ff9857710", "model": "google/gemini-3.6-flash", "case_id": "banking-ios/clean/grounding/transaction-search", "status": "success", "wall_time_ms": 1125.072, "cost": 0.002076, "schemaValid": true, "pointHit": true, "boxIoU": 0.8606775633691806, "boxIoUAt50": true }, { "attempt_id": "aede1f3096200e5d252bf29f", "model": "google/gemini-3.6-flash", "case_id": "banking-ios/clean/grounding/hide-balance", "status": "success", "wall_time_ms": 1563.019, "cost": 0.0020685, "schemaValid": true, "pointHit": true, "boxIoU": 0.03312031162027591, "boxIoUAt50": false }, { "attempt_id": "0719ea4e87d66425c1a7cb78", "model": "google/gemini-3.6-flash", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "status": "invalid_response", "wall_time_ms": 3485.663, "cost": 0.004671, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "d1c29ba7e31b45d7003b12a0", "model": "google/gemini-3.6-flash", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "status": "success", "wall_time_ms": 1293.231, "cost": 0.002082, "schemaValid": true, "pointHit": true, "boxIoU": 0.9725320939824987, "boxIoUAt50": true }, { "attempt_id": "0c23f9ca2bcdf238f747757a", "model": "google/gemini-3.6-flash", "case_id": "flight-pass-ios/clean/grounding/check-in", "status": "success", "wall_time_ms": 1198.286, "cost": 0.002076, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "a14a05cb0a946fabf1c36a46", "model": "google/gemini-3.6-flash", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "status": "invalid_response", "wall_time_ms": 3296.711, "cost": 0.0046635, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "e29bd052426e251080e462ab", "model": "google/gemini-3.6-flash", "case_id": "smart-home-android/clean/grounding/thermostat-card", "status": "success", "wall_time_ms": 1558.857, "cost": 0.0020775, "schemaValid": true, "pointHit": true, "boxIoU": 0.986720876111822, "boxIoUAt50": true }, { "attempt_id": "662d20730f45423a2e3e2f0d", "model": "google/gemini-3.6-flash", "case_id": "smart-home-android/clean/grounding/add-device", "status": "success", "wall_time_ms": 1773.689, "cost": 0.0021735, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "eecaca344b1c88c794c87bbb", "model": "google/gemini-3.6-flash", "case_id": "smart-home-android/clean/grounding/more", "status": "success", "wall_time_ms": 1344.212, "cost": 0.002175, "schemaValid": true, "pointHit": true, "boxIoU": 0.4452153706667883, "boxIoUAt50": false }, { "attempt_id": "5bae698a58da1586f409450a", "model": "google/gemini-3.6-flash", "case_id": "smart-home-android/clean/grounding/room-search", "status": "success", "wall_time_ms": 1192.967, "cost": 0.002076, "schemaValid": true, "pointHit": true, "boxIoU": 0.8661905315663693, "boxIoUAt50": true } ] }, "openai/gpt-5.6-sol": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "metrics": { "pointAccuracy": 0.9583333333333334, "meanIoU": 0.6768970507750426, "iouAt50": 0.7083333333333334, "schemaValidRate": 1 }, "repeat_consistency": null, "wall_time_ms": [ 3079.495, 3672.496, 2119.468, 3582.094, 2022.172, 3697.298, 4203.433, 2641.448, 3765.183, 1417.215, 3958.254, 2078.831, 2645.214, 3231.992, 3875.373, 6713.742, 2702.362, 1695.896, 1521.004, 7339.229, 1680.828, 2303.87, 4319.527, 3150.245 ], "total_cost": 0.24532749999999995, "attempts": [ { "attempt_id": "6f10ae292ea6ed409d9d7d59", "model": "openai/gpt-5.6-sol", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "status": "success", "wall_time_ms": 3079.495, "cost": 0.01288625, "schemaValid": true, "pointHit": true, "boxIoU": 0.9934718156600124, "boxIoUAt50": true }, { "attempt_id": "86bcfeeb5bd1b768f4c4018e", "model": "openai/gpt-5.6-sol", "case_id": "commerce-catalog-desktop/clean/grounding/search", "status": "success", "wall_time_ms": 3672.496, "cost": 0.01477625, "schemaValid": true, "pointHit": true, "boxIoU": 0.7009230818820404, "boxIoUAt50": true }, { "attempt_id": "efeddae9afcead4ea0145c67", "model": "openai/gpt-5.6-sol", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "status": "success", "wall_time_ms": 2119.468, "cost": 0.01258625, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "0bff8bad02f8e92fcb4819e2", "model": "openai/gpt-5.6-sol", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "status": "success", "wall_time_ms": 3582.094, "cost": 0.0155025, "schemaValid": true, "pointHit": true, "boxIoU": 0.06907206127460082, "boxIoUAt50": false }, { "attempt_id": "27f6d4e4f37bcf64e642f970", "model": "openai/gpt-5.6-sol", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "status": "success", "wall_time_ms": 2022.172, "cost": 0.01258, "schemaValid": true, "pointHit": true, "boxIoU": 0.8209025848659529, "boxIoUAt50": true }, { "attempt_id": "d6bcbd4e70dbf0159018d028", "model": "openai/gpt-5.6-sol", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "status": "success", "wall_time_ms": 3697.298, "cost": 0.01552625, "schemaValid": true, "pointHit": true, "boxIoU": 0.4391242072380626, "boxIoUAt50": false }, { "attempt_id": "cbe8a9d90caf11c1c057a1f0", "model": "openai/gpt-5.6-sol", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "status": "success", "wall_time_ms": 4203.433, "cost": 0.01666, "schemaValid": true, "pointHit": true, "boxIoU": 0.9257175587027299, "boxIoUAt50": true }, { "attempt_id": "91192cc98485299e16016bbd", "model": "openai/gpt-5.6-sol", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "status": "success", "wall_time_ms": 2641.448, "cost": 0.01447625, "schemaValid": true, "pointHit": true, "boxIoU": 0.9890804852125081, "boxIoUAt50": true }, { "attempt_id": "2cf38d41cd5f15c8ee9c13e7", "model": "openai/gpt-5.6-sol", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "status": "success", "wall_time_ms": 3765.183, "cost": 0.01807625, "schemaValid": true, "pointHit": true, "boxIoU": 0.03974687097878154, "boxIoUAt50": false }, { "attempt_id": "7d33e6397d85ab56ea58f035", "model": "openai/gpt-5.6-sol", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "status": "success", "wall_time_ms": 1417.215, "cost": 0.01258625, "schemaValid": true, "pointHit": true, "boxIoU": 0.9241496170948116, "boxIoUAt50": true }, { "attempt_id": "4834e8859f7327c9e41171a5", "model": "openai/gpt-5.6-sol", "case_id": "iot-control-desktop/clean/grounding/pump-card", "status": "success", "wall_time_ms": 3958.254, "cost": 0.014285, "schemaValid": true, "pointHit": true, "boxIoU": 0.9757049108661706, "boxIoUAt50": true }, { "attempt_id": "d06322701b302b06dd15af56", "model": "openai/gpt-5.6-sol", "case_id": "iot-control-desktop/clean/grounding/restart-device", "status": "success", "wall_time_ms": 2078.831, "cost": 0.01258625, "schemaValid": true, "pointHit": false, "boxIoU": 0.28753954953094657, "boxIoUAt50": false }, { "attempt_id": "d43f6c5605fc9030a2215774", "model": "openai/gpt-5.6-sol", "case_id": "banking-ios/clean/grounding/transfer", "status": "success", "wall_time_ms": 2645.214, "cost": 0.00457, "schemaValid": true, "pointHit": true, "boxIoU": 0.3543952703962102, "boxIoUAt50": false }, { "attempt_id": "50b366da901c930304592f6b", "model": "openai/gpt-5.6-sol", "case_id": "banking-ios/clean/grounding/profile", "status": "success", "wall_time_ms": 3231.992, "cost": 0.006435, "schemaValid": true, "pointHit": true, "boxIoU": 0.9631189651159091, "boxIoUAt50": true }, { "attempt_id": "38cb5b23b76481b94a239942", "model": "openai/gpt-5.6-sol", "case_id": "banking-ios/clean/grounding/transaction-search", "status": "success", "wall_time_ms": 3875.373, "cost": 0.006555, "schemaValid": true, "pointHit": true, "boxIoU": 0.848827687345642, "boxIoUAt50": true }, { "attempt_id": "21cdd3a9a9b330cce96a906a", "model": "openai/gpt-5.6-sol", "case_id": "banking-ios/clean/grounding/hide-balance", "status": "success", "wall_time_ms": 6713.742, "cost": 0.007785, "schemaValid": true, "pointHit": true, "boxIoU": 0.20108995068157762, "boxIoUAt50": false }, { "attempt_id": "221270963338a99f2b65668c", "model": "openai/gpt-5.6-sol", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "status": "success", "wall_time_ms": 2702.362, "cost": 0.006915, "schemaValid": true, "pointHit": true, "boxIoU": 0.728029984198174, "boxIoUAt50": true }, { "attempt_id": "23ad85de3d74d2e4a04332a9", "model": "openai/gpt-5.6-sol", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "status": "success", "wall_time_ms": 1695.896, "cost": 0.00459, "schemaValid": true, "pointHit": true, "boxIoU": 0.9170285429977784, "boxIoUAt50": true }, { "attempt_id": "f196eaa332fd580d4fec0df5", "model": "openai/gpt-5.6-sol", "case_id": "flight-pass-ios/clean/grounding/check-in", "status": "success", "wall_time_ms": 1521.004, "cost": 0.004575, "schemaValid": true, "pointHit": true, "boxIoU": 0.8033160852012929, "boxIoUAt50": true }, { "attempt_id": "03038ab26a2b26ab4af8db34", "model": "openai/gpt-5.6-sol", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "status": "success", "wall_time_ms": 7339.229, "cost": 0.011265, "schemaValid": true, "pointHit": true, "boxIoU": 0.7279606031598126, "boxIoUAt50": true }, { "attempt_id": "07bead8e17b18a2b2a6f59f5", "model": "openai/gpt-5.6-sol", "case_id": "smart-home-android/clean/grounding/thermostat-card", "status": "success", "wall_time_ms": 1680.828, "cost": 0.00458, "schemaValid": true, "pointHit": true, "boxIoU": 0.45791428367453163, "boxIoUAt50": false }, { "attempt_id": "1b0e6ef7f6c97a867ed84381", "model": "openai/gpt-5.6-sol", "case_id": "smart-home-android/clean/grounding/add-device", "status": "success", "wall_time_ms": 2303.87, "cost": 0.004575, "schemaValid": true, "pointHit": true, "boxIoU": 0.5889037387939656, "boxIoUAt50": true }, { "attempt_id": "ae527cfdfb52e2451e4c50c3", "model": "openai/gpt-5.6-sol", "case_id": "smart-home-android/clean/grounding/more", "status": "success", "wall_time_ms": 4319.527, "cost": 0.00458, "schemaValid": true, "pointHit": true, "boxIoU": 0.8003891525912613, "boxIoUAt50": true }, { "attempt_id": "a1f807c80e1c170726f2bc23", "model": "openai/gpt-5.6-sol", "case_id": "smart-home-android/clean/grounding/room-search", "status": "success", "wall_time_ms": 3150.245, "cost": 0.006375, "schemaValid": true, "pointHit": true, "boxIoU": 0.8644842570776636, "boxIoUAt50": true } ] }, "openai/gpt-5.6-luna": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "metrics": { "pointAccuracy": 1, "meanIoU": 0.770578890564131, "iouAt50": 0.7916666666666666, "schemaValidRate": 1 }, "repeat_consistency": null, "wall_time_ms": [ 2904.008, 2162.391, 2898.307, 2861.531, 3857.921, 2375.449, 2938.843, 2335.164, 3774.799, 5375.267, 2832.293, 2698.008, 2618.537, 11859.418, 2027.099, 4255.449, 4746.029, 8700.997, 2659.089, 3443.336, 2403.222, 1060.39, 2133.529, 2181.935 ], "total_cost": 0.005701550000000001, "attempts": [ { "attempt_id": "55d6ec30ef35e5dfbeef16ba", "model": "openai/gpt-5.6-luna", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "status": "success", "wall_time_ms": 2904.008, "cost": 0.000290725, "schemaValid": true, "pointHit": true, "boxIoU": 0.9734577622286735, "boxIoUAt50": true }, { "attempt_id": "9b87889108bb3f761c8a14ae", "model": "openai/gpt-5.6-luna", "case_id": "commerce-catalog-desktop/clean/grounding/search", "status": "success", "wall_time_ms": 2162.391, "cost": 0.000312325, "schemaValid": true, "pointHit": true, "boxIoU": 0.9675787247276455, "boxIoUAt50": true }, { "attempt_id": "35088b88594f6e8a80720e44", "model": "openai/gpt-5.6-luna", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "status": "success", "wall_time_ms": 2898.307, "cost": 0.000293125, "schemaValid": true, "pointHit": true, "boxIoU": 0.8246379540605832, "boxIoUAt50": true }, { "attempt_id": "c69f2acefcb8c6afd31a5f43", "model": "openai/gpt-5.6-luna", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "status": "success", "wall_time_ms": 2861.531, "cost": 0.00032805, "schemaValid": true, "pointHit": true, "boxIoU": 0.06561845821087078, "boxIoUAt50": false }, { "attempt_id": "3d6b14a8e856ec541ff1ed4a", "model": "openai/gpt-5.6-luna", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "status": "success", "wall_time_ms": 3857.921, "cost": 0.000296, "schemaValid": true, "pointHit": true, "boxIoU": 0.9782124823785875, "boxIoUAt50": true }, { "attempt_id": "c2947e6c727cd49897eb52d0", "model": "openai/gpt-5.6-luna", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "status": "success", "wall_time_ms": 2375.449, "cost": 0.000314125, "schemaValid": true, "pointHit": true, "boxIoU": 0.4171679968761595, "boxIoUAt50": false }, { "attempt_id": "7f26b427db2b00ee43f904e3", "model": "openai/gpt-5.6-luna", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "status": "success", "wall_time_ms": 2938.843, "cost": 0.0003458, "schemaValid": true, "pointHit": true, "boxIoU": 0.898220200097302, "boxIoUAt50": true }, { "attempt_id": "c6c2288235d1f241b7236ce8", "model": "openai/gpt-5.6-luna", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "status": "success", "wall_time_ms": 2335.164, "cost": 0.000305725, "schemaValid": true, "pointHit": true, "boxIoU": 0.9828747723510349, "boxIoUAt50": true }, { "attempt_id": "1d50c9f2c2f1d7d89842bec4", "model": "openai/gpt-5.6-luna", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "status": "success", "wall_time_ms": 3774.799, "cost": 0.000334525, "schemaValid": true, "pointHit": true, "boxIoU": 0.04063441208484528, "boxIoUAt50": false }, { "attempt_id": "1d55fb95d5a7e5d97a8fa36d", "model": "openai/gpt-5.6-luna", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "status": "success", "wall_time_ms": 5375.267, "cost": 0.000302725, "schemaValid": true, "pointHit": true, "boxIoU": 0.9908265557744141, "boxIoUAt50": true }, { "attempt_id": "8bd81ef2fe4be09cb02b1ea5", "model": "openai/gpt-5.6-luna", "case_id": "iot-control-desktop/clean/grounding/pump-card", "status": "success", "wall_time_ms": 2832.293, "cost": 0.0003259, "schemaValid": true, "pointHit": true, "boxIoU": 0.9757049108661706, "boxIoUAt50": true }, { "attempt_id": "a33222074d2eb27f8f941d12", "model": "openai/gpt-5.6-luna", "case_id": "iot-control-desktop/clean/grounding/restart-device", "status": "success", "wall_time_ms": 2698.008, "cost": 0.000306925, "schemaValid": true, "pointHit": true, "boxIoU": 0.9722465288420963, "boxIoUAt50": true }, { "attempt_id": "02a11565d0cffe5d122f8e5e", "model": "openai/gpt-5.6-luna", "case_id": "banking-ios/clean/grounding/transfer", "status": "success", "wall_time_ms": 2618.537, "cost": 0.0001382, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "895dcc587593a396890e9d01", "model": "openai/gpt-5.6-luna", "case_id": "banking-ios/clean/grounding/profile", "status": "success", "wall_time_ms": 11859.418, "cost": 0.0001371, "schemaValid": true, "pointHit": true, "boxIoU": 0.9519044965694483, "boxIoUAt50": true }, { "attempt_id": "40d7edabf2ed96dde3d06aae", "model": "openai/gpt-5.6-luna", "case_id": "banking-ios/clean/grounding/transaction-search", "status": "success", "wall_time_ms": 2027.099, "cost": 0.0001395, "schemaValid": true, "pointHit": true, "boxIoU": 0.8476522453114688, "boxIoUAt50": true }, { "attempt_id": "26be5510157f1d21b7749861", "model": "openai/gpt-5.6-luna", "case_id": "banking-ios/clean/grounding/hide-balance", "status": "success", "wall_time_ms": 4255.449, "cost": 0.0002307, "schemaValid": true, "pointHit": true, "boxIoU": 0.03248338255065523, "boxIoUAt50": false }, { "attempt_id": "e239a6454121eac31d9f584b", "model": "openai/gpt-5.6-luna", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "status": "success", "wall_time_ms": 4746.029, "cost": 0.0002919, "schemaValid": true, "pointHit": true, "boxIoU": 0.12654100576423857, "boxIoUAt50": false }, { "attempt_id": "7af4201bdb37b720b9015841", "model": "openai/gpt-5.6-luna", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "status": "success", "wall_time_ms": 8700.997, "cost": 0.000135, "schemaValid": true, "pointHit": true, "boxIoU": 0.986342221925507, "boxIoUAt50": true }, { "attempt_id": "7b1d0cabab8c41d062e18485", "model": "openai/gpt-5.6-luna", "case_id": "flight-pass-ios/clean/grounding/check-in", "status": "success", "wall_time_ms": 2659.089, "cost": 0.0001335, "schemaValid": true, "pointHit": true, "boxIoU": 0.9873400368524146, "boxIoUAt50": true }, { "attempt_id": "b19e3b2df7899948d0d95f43", "model": "openai/gpt-5.6-luna", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "status": "success", "wall_time_ms": 3443.336, "cost": 0.0002235, "schemaValid": true, "pointHit": true, "boxIoU": 0.7279606031598126, "boxIoUAt50": true }, { "attempt_id": "4eed0077fedea7452c5e3abe", "model": "openai/gpt-5.6-luna", "case_id": "smart-home-android/clean/grounding/thermostat-card", "status": "success", "wall_time_ms": 2403.222, "cost": 0.000133, "schemaValid": true, "pointHit": true, "boxIoU": 0.9930474343914654, "boxIoUAt50": true }, { "attempt_id": "e70b2ed700d028ea9948015f", "model": "openai/gpt-5.6-luna", "case_id": "smart-home-android/clean/grounding/add-device", "status": "success", "wall_time_ms": 1060.39, "cost": 0.0000915, "schemaValid": true, "pointHit": true, "boxIoU": 0.9598488211838173, "boxIoUAt50": true }, { "attempt_id": "78ee0e61d0b9118389cdf7d2", "model": "openai/gpt-5.6-luna", "case_id": "smart-home-android/clean/grounding/more", "status": "success", "wall_time_ms": 2133.529, "cost": 0.0001552, "schemaValid": true, "pointHit": true, "boxIoU": 0.9519044965694483, "boxIoUAt50": true }, { "attempt_id": "dddf5dcd437e4c19ecaf34fe", "model": "openai/gpt-5.6-luna", "case_id": "smart-home-android/clean/grounding/room-search", "status": "success", "wall_time_ms": 2181.935, "cost": 0.0001365, "schemaValid": true, "pointHit": true, "boxIoU": 0.8644842570776636, "boxIoUAt50": true } ] }, "anthropic/claude-sonnet-5": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "metrics": { "pointAccuracy": 0.20833333333333334, "meanIoU": 0.04322858121619134, "iouAt50": 0, "schemaValidRate": 0.8333333333333334 }, "repeat_consistency": null, "wall_time_ms": [ 5521.263, 4319.351, 3582.395, 3759.912, 3637.049, 3932.465, 3713.157, 3345.975, 3579.928, 3611.359, 3693.142, 4200.597, 2963.37, 3674.239, 4065.625, 3615.257, 3969.868, 5428.079, 3628.569, 4131.826, 3598.231, 4273.514, 3978.166, 3548.056 ], "total_cost": 0.09897, "attempts": [ { "attempt_id": "a72fe006942ba9f6a8c06c5f", "model": "anthropic/claude-sonnet-5", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "status": "invalid_response", "wall_time_ms": 5521.263, "cost": 0.005418, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "82ee266d814325b798ff9fc9", "model": "anthropic/claude-sonnet-5", "case_id": "commerce-catalog-desktop/clean/grounding/search", "status": "invalid_response", "wall_time_ms": 4319.351, "cost": 0.005418, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "432725f506ef41905834423a", "model": "anthropic/claude-sonnet-5", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "status": "success", "wall_time_ms": 3582.395, "cost": 0.005402, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "e78e489447530907881a28a9", "model": "anthropic/claude-sonnet-5", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "status": "success", "wall_time_ms": 3759.912, "cost": 0.005406, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "f0f22828a6c0fbb2d6854c9f", "model": "anthropic/claude-sonnet-5", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "status": "success", "wall_time_ms": 3637.049, "cost": 0.005394, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "17948e53326c6513ef78023a", "model": "anthropic/claude-sonnet-5", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "status": "success", "wall_time_ms": 3932.465, "cost": 0.005404, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "d5a79bf82f33b237c8ae31b0", "model": "anthropic/claude-sonnet-5", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "status": "success", "wall_time_ms": 3713.157, "cost": 0.0054, "schemaValid": true, "pointHit": true, "boxIoU": 0.18414976162096383, "boxIoUAt50": false }, { "attempt_id": "a8e4446477fc0c8a47e1a366", "model": "anthropic/claude-sonnet-5", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "status": "success", "wall_time_ms": 3345.975, "cost": 0.005404, "schemaValid": true, "pointHit": true, "boxIoU": 0.32570041602579547, "boxIoUAt50": false }, { "attempt_id": "9993b6755675dc44fe5da428", "model": "anthropic/claude-sonnet-5", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "status": "success", "wall_time_ms": 3579.928, "cost": 0.005408, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "122a3e7d19dc29bafd9c7f09", "model": "anthropic/claude-sonnet-5", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "status": "success", "wall_time_ms": 3611.359, "cost": 0.005404, "schemaValid": true, "pointHit": true, "boxIoU": 0.3664239838033855, "boxIoUAt50": false }, { "attempt_id": "541e0e9b440bdfc21c4b0278", "model": "anthropic/claude-sonnet-5", "case_id": "iot-control-desktop/clean/grounding/pump-card", "status": "invalid_response", "wall_time_ms": 3693.142, "cost": 0.005422, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "fa8c1d626b750afa7cfa66c2", "model": "anthropic/claude-sonnet-5", "case_id": "iot-control-desktop/clean/grounding/restart-device", "status": "invalid_response", "wall_time_ms": 4200.597, "cost": 0.00542, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "220bf00419dc47eff4069d5b", "model": "anthropic/claude-sonnet-5", "case_id": "banking-ios/clean/grounding/transfer", "status": "success", "wall_time_ms": 2963.37, "cost": 0.002836, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "107ba283a0e553b3a028f286", "model": "anthropic/claude-sonnet-5", "case_id": "banking-ios/clean/grounding/profile", "status": "success", "wall_time_ms": 3674.239, "cost": 0.002836, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "c6b440c52d947c18cba4dccd", "model": "anthropic/claude-sonnet-5", "case_id": "banking-ios/clean/grounding/transaction-search", "status": "success", "wall_time_ms": 4065.625, "cost": 0.002838, "schemaValid": true, "pointHit": true, "boxIoU": 0.07281972837129133, "boxIoUAt50": false }, { "attempt_id": "075d94a5b9f63a02c646adc2", "model": "anthropic/claude-sonnet-5", "case_id": "banking-ios/clean/grounding/hide-balance", "status": "success", "wall_time_ms": 3615.257, "cost": 0.002842, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "976f5c9a1370df017f782620", "model": "anthropic/claude-sonnet-5", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "status": "success", "wall_time_ms": 3969.868, "cost": 0.002842, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "b7935b3248928adc73cb7652", "model": "anthropic/claude-sonnet-5", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "status": "success", "wall_time_ms": 5428.079, "cost": 0.00285, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "29c6efee2091f997b9cd1571", "model": "anthropic/claude-sonnet-5", "case_id": "flight-pass-ios/clean/grounding/check-in", "status": "success", "wall_time_ms": 3628.569, "cost": 0.002834, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "e97fbef48d99a62829dcc214", "model": "anthropic/claude-sonnet-5", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "status": "success", "wall_time_ms": 4131.826, "cost": 0.002842, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "d64d05e72a07e2aabfad9ff8", "model": "anthropic/claude-sonnet-5", "case_id": "smart-home-android/clean/grounding/thermostat-card", "status": "success", "wall_time_ms": 3598.231, "cost": 0.002842, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "02329ee536b934801236602a", "model": "anthropic/claude-sonnet-5", "case_id": "smart-home-android/clean/grounding/add-device", "status": "success", "wall_time_ms": 4273.514, "cost": 0.002834, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "c893de3510251088495168bc", "model": "anthropic/claude-sonnet-5", "case_id": "smart-home-android/clean/grounding/more", "status": "success", "wall_time_ms": 3978.166, "cost": 0.002836, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "3117d402bb8816a8010fa9d4", "model": "anthropic/claude-sonnet-5", "case_id": "smart-home-android/clean/grounding/room-search", "status": "success", "wall_time_ms": 3548.056, "cost": 0.002838, "schemaValid": true, "pointHit": true, "boxIoU": 0.08839205936715591, "boxIoUAt50": false } ] }, "anthropic/claude-opus-5": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "metrics": { "pointAccuracy": 0.4583333333333333, "meanIoU": 0.3139966886707624, "iouAt50": 0.3333333333333333, "schemaValidRate": 0.875 }, "repeat_consistency": null, "wall_time_ms": [ 6680.762, 4122.19, 3451.233, 3825.511, 4294.557, 4693.194, 4196.418, 4852.053, 3869.552, 3757.365, 4460.078, 4108.102, 4648.59, 4012.529, 5872.892, 4956.129, 3405.76, 3394.799, 3511.844, 3904.579, 3706.101, 3887.202, 4103.154, 3979.162 ], "total_cost": 0.24740000000000004, "attempts": [ { "attempt_id": "094a8a177bc917f272e9eaf9", "model": "anthropic/claude-opus-5", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "status": "success", "wall_time_ms": 6680.762, "cost": 0.013495, "schemaValid": true, "pointHit": true, "boxIoU": 0.9164451618426002, "boxIoUAt50": true }, { "attempt_id": "319934a5941e35c5cab8f7d9", "model": "anthropic/claude-opus-5", "case_id": "commerce-catalog-desktop/clean/grounding/search", "status": "success", "wall_time_ms": 4122.19, "cost": 0.013495, "schemaValid": true, "pointHit": true, "boxIoU": 0.5936258658255639, "boxIoUAt50": true }, { "attempt_id": "ed7d80aa6b8d1048823a0f28", "model": "anthropic/claude-opus-5", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "status": "success", "wall_time_ms": 3451.233, "cost": 0.013505, "schemaValid": true, "pointHit": true, "boxIoU": 0.8612885297966091, "boxIoUAt50": true }, { "attempt_id": "3545485e7db363bc3f1bc9f6", "model": "anthropic/claude-opus-5", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "status": "success", "wall_time_ms": 3825.511, "cost": 0.013515, "schemaValid": true, "pointHit": true, "boxIoU": 0.07108666306177668, "boxIoUAt50": false }, { "attempt_id": "07521c3956fc6c0abc1d7d35", "model": "anthropic/claude-opus-5", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "status": "success", "wall_time_ms": 4294.557, "cost": 0.013485, "schemaValid": true, "pointHit": true, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "ec74d7deefd9d6bed012ac8c", "model": "anthropic/claude-opus-5", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "status": "success", "wall_time_ms": 4693.194, "cost": 0.01351, "schemaValid": true, "pointHit": true, "boxIoU": 0.4820703573026539, "boxIoUAt50": false }, { "attempt_id": "275ac78b047e16503a52f203", "model": "anthropic/claude-opus-5", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "status": "success", "wall_time_ms": 4196.418, "cost": 0.0135, "schemaValid": true, "pointHit": true, "boxIoU": 0.82973201765022, "boxIoUAt50": true }, { "attempt_id": "3451745528ef8998dabff370", "model": "anthropic/claude-opus-5", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "status": "success", "wall_time_ms": 4852.053, "cost": 0.01351, "schemaValid": true, "pointHit": true, "boxIoU": 0.982195935868697, "boxIoUAt50": true }, { "attempt_id": "ac73ab0ca55b5f1e68f0d663", "model": "anthropic/claude-opus-5", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "status": "success", "wall_time_ms": 3869.552, "cost": 0.01352, "schemaValid": true, "pointHit": true, "boxIoU": 0.59817333653868, "boxIoUAt50": true }, { "attempt_id": "7d111c16ce32ad3e212f0353", "model": "anthropic/claude-opus-5", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "status": "success", "wall_time_ms": 3757.365, "cost": 0.01351, "schemaValid": true, "pointHit": true, "boxIoU": 0.8127633351718062, "boxIoUAt50": true }, { "attempt_id": "83e30b876f3542d7ed47eba0", "model": "anthropic/claude-opus-5", "case_id": "iot-control-desktop/clean/grounding/pump-card", "status": "success", "wall_time_ms": 4460.078, "cost": 0.01353, "schemaValid": true, "pointHit": false, "boxIoU": 0.15626421079452232, "boxIoUAt50": false }, { "attempt_id": "6c244361373a80fc9c68c0ea", "model": "anthropic/claude-opus-5", "case_id": "iot-control-desktop/clean/grounding/restart-device", "status": "success", "wall_time_ms": 4108.102, "cost": 0.0135, "schemaValid": true, "pointHit": true, "boxIoU": 0.9383796593779474, "boxIoUAt50": true }, { "attempt_id": "a19ef6cb5611defa38e35b40", "model": "anthropic/claude-opus-5", "case_id": "banking-ios/clean/grounding/transfer", "status": "invalid_response", "wall_time_ms": 4648.59, "cost": 0.00714, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "b05f919222c185cf74ad6ac8", "model": "anthropic/claude-opus-5", "case_id": "banking-ios/clean/grounding/profile", "status": "success", "wall_time_ms": 4012.529, "cost": 0.00709, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "21cad73359bcfe2d8b57d334", "model": "anthropic/claude-opus-5", "case_id": "banking-ios/clean/grounding/transaction-search", "status": "success", "wall_time_ms": 5872.892, "cost": 0.007095, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "863a648fdbde0867d17ff2c2", "model": "anthropic/claude-opus-5", "case_id": "banking-ios/clean/grounding/hide-balance", "status": "success", "wall_time_ms": 4956.129, "cost": 0.007105, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "65a2d734a1ac6fbab470e57c", "model": "anthropic/claude-opus-5", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "status": "invalid_response", "wall_time_ms": 3405.76, "cost": 0.007155, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "27104b9c88a432db9b42ad37", "model": "anthropic/claude-opus-5", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "status": "success", "wall_time_ms": 3394.799, "cost": 0.007125, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "2d8d635d7b365ac4956dc416", "model": "anthropic/claude-opus-5", "case_id": "flight-pass-ios/clean/grounding/check-in", "status": "success", "wall_time_ms": 3511.844, "cost": 0.007085, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "a0f80bcf869f819194432a0d", "model": "anthropic/claude-opus-5", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "status": "success", "wall_time_ms": 3904.579, "cost": 0.007105, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "e5cc3a8e89f0a85a2737f5af", "model": "anthropic/claude-opus-5", "case_id": "smart-home-android/clean/grounding/thermostat-card", "status": "success", "wall_time_ms": 3706.101, "cost": 0.007105, "schemaValid": true, "pointHit": false, "boxIoU": 0.29389545486722135, "boxIoUAt50": false }, { "attempt_id": "11d054945ec0cb81cbb0d1c3", "model": "anthropic/claude-opus-5", "case_id": "smart-home-android/clean/grounding/add-device", "status": "invalid_response", "wall_time_ms": 3887.202, "cost": 0.007135, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "6b923c4989f8af9786a071ad", "model": "anthropic/claude-opus-5", "case_id": "smart-home-android/clean/grounding/more", "status": "success", "wall_time_ms": 4103.154, "cost": 0.00709, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "03e36a78b39581492e04fac0", "model": "anthropic/claude-opus-5", "case_id": "smart-home-android/clean/grounding/room-search", "status": "success", "wall_time_ms": 3979.162, "cost": 0.007095, "schemaValid": true, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false } ] }, "google/gemini-3.1-pro-preview": { "transport_attempt_count": 24, "response_count": 24, "request_failure_count": 0, "metrics": { "pointAccuracy": 0.875, "meanIoU": 0.6316910569751387, "iouAt50": 0.5833333333333334, "schemaValidRate": 0.875 }, "repeat_consistency": null, "wall_time_ms": [ 11667.216, 3773.697, 6040.544, 3339.043, 3780.606, 3001.476, 3548.756, 8645.31, 3713.037, 6472.185, 27789.668, 22847.746, 2714.075, 9113.302, 37418.122, 52212.435, 3509.957, 2879.705, 6646.308, 35165.545, 4407.107, 3347.916, 2890.414, 20549.805 ], "total_cost": 0.072566, "attempts": [ { "attempt_id": "2dc626f8e31822c7d56939ba", "model": "google/gemini-3.1-pro-preview", "case_id": "commerce-catalog-desktop/clean/grounding/add-product", "status": "success", "wall_time_ms": 11667.216, "cost": 0.003078, "schemaValid": true, "pointHit": true, "boxIoU": 0.9355874913498428, "boxIoUAt50": true }, { "attempt_id": "0d89f5b5732159302940f5ef", "model": "google/gemini-3.1-pro-preview", "case_id": "commerce-catalog-desktop/clean/grounding/search", "status": "success", "wall_time_ms": 3773.697, "cost": 0.003006, "schemaValid": true, "pointHit": true, "boxIoU": 0.14086547908222144, "boxIoUAt50": false }, { "attempt_id": "5d49640e49b00d5f59a8e58d", "model": "google/gemini-3.1-pro-preview", "case_id": "commerce-catalog-desktop/clean/grounding/catalog-query", "status": "success", "wall_time_ms": 6040.544, "cost": 0.003042, "schemaValid": true, "pointHit": true, "boxIoU": 0.8612885297966091, "boxIoUAt50": true }, { "attempt_id": "c2356f36c6116d3dd2d84e04", "model": "google/gemini-3.1-pro-preview", "case_id": "commerce-catalog-desktop/clean/grounding/stock-filter", "status": "success", "wall_time_ms": 3339.043, "cost": 0.00302, "schemaValid": true, "pointHit": true, "boxIoU": 0.48321662866689485, "boxIoUAt50": false }, { "attempt_id": "6de7b7c0b69a272a626805e5", "model": "google/gemini-3.1-pro-preview", "case_id": "analytics-dashboard-desktop/clean/grounding/search-field", "status": "success", "wall_time_ms": 3780.606, "cost": 0.003064, "schemaValid": true, "pointHit": true, "boxIoU": 0.9656445141519203, "boxIoUAt50": true }, { "attempt_id": "08a1155220620bd9350d7f16", "model": "google/gemini-3.1-pro-preview", "case_id": "analytics-dashboard-desktop/clean/grounding/compare-toggle", "status": "success", "wall_time_ms": 3001.476, "cost": 0.003054, "schemaValid": true, "pointHit": true, "boxIoU": 0.40146436681240144, "boxIoUAt50": false }, { "attempt_id": "6717dccbb96e03486ff11904", "model": "google/gemini-3.1-pro-preview", "case_id": "analytics-dashboard-desktop/clean/grounding/audience-tab", "status": "success", "wall_time_ms": 3548.756, "cost": 0.003028, "schemaValid": true, "pointHit": true, "boxIoU": 0.9597851732559296, "boxIoUAt50": true }, { "attempt_id": "949face3ca55a94aa31217a5", "model": "google/gemini-3.1-pro-preview", "case_id": "analytics-dashboard-desktop/clean/grounding/revenue-card", "status": "success", "wall_time_ms": 8645.31, "cost": 0.003054, "schemaValid": true, "pointHit": true, "boxIoU": 0.9906109155726818, "boxIoUAt50": true }, { "attempt_id": "403ce0879dde8354ffcbdae7", "model": "google/gemini-3.1-pro-preview", "case_id": "iot-control-desktop/clean/grounding/auto-mode", "status": "success", "wall_time_ms": 3713.037, "cost": 0.003054, "schemaValid": true, "pointHit": true, "boxIoU": 0.42351450271881735, "boxIoUAt50": false }, { "attempt_id": "a66cdb37ece9c42b4d5bc4d8", "model": "google/gemini-3.1-pro-preview", "case_id": "iot-control-desktop/clean/grounding/devices-nav", "status": "success", "wall_time_ms": 6472.185, "cost": 0.003018, "schemaValid": true, "pointHit": true, "boxIoU": 0.9800789308622267, "boxIoUAt50": true }, { "attempt_id": "da385a5c33f5bf8d1ec98be7", "model": "google/gemini-3.1-pro-preview", "case_id": "iot-control-desktop/clean/grounding/pump-card", "status": "invalid_response", "wall_time_ms": 27789.668, "cost": 0.00306, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "d1e60f12bce02f9aa08daf1c", "model": "google/gemini-3.1-pro-preview", "case_id": "iot-control-desktop/clean/grounding/restart-device", "status": "invalid_response", "wall_time_ms": 22847.746, "cost": 0.00303, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "b67c832275c8ef95f3a8600a", "model": "google/gemini-3.1-pro-preview", "case_id": "banking-ios/clean/grounding/transfer", "status": "success", "wall_time_ms": 2714.075, "cost": 0.003, "schemaValid": true, "pointHit": true, "boxIoU": 0.9764677825269023, "boxIoUAt50": true }, { "attempt_id": "d3c838e3fe4329ea28428ab7", "model": "google/gemini-3.1-pro-preview", "case_id": "banking-ios/clean/grounding/profile", "status": "invalid_response", "wall_time_ms": 9113.302, "cost": 0.002978, "schemaValid": false, "pointHit": false, "boxIoU": 0, "boxIoUAt50": false }, { "attempt_id": "5a25f38db212c2c585282dd9", "model": "google/gemini-3.1-pro-preview", "case_id": "banking-ios/clean/grounding/transaction-search", "status": "success", "wall_time_ms": 37418.122, "cost": 0.003002, "schemaValid": true, "pointHit": true, "boxIoU": 0.8394664765948328, "boxIoUAt50": true }, { "attempt_id": "4fed705d27e31db1f6fa1fb9", "model": "google/gemini-3.1-pro-preview", "case_id": "banking-ios/clean/grounding/hide-balance", "status": "success", "wall_time_ms": 52212.435, "cost": 0.003014, "schemaValid": true, "pointHit": true, "boxIoU": 0.27213471892803986, "boxIoUAt50": false }, { "attempt_id": "700b464ab07fda807134aa61", "model": "google/gemini-3.1-pro-preview", "case_id": "flight-pass-ios/clean/grounding/trips-tab", "status": "success", "wall_time_ms": 3509.957, "cost": 0.003014, "schemaValid": true, "pointHit": true, "boxIoU": 0.12230203906990303, "boxIoUAt50": false }, { "attempt_id": "e35426e4b4185500394b9df3", "model": "google/gemini-3.1-pro-preview", "case_id": "flight-pass-ios/clean/grounding/boarding-pass", "status": "success", "wall_time_ms": 2879.705, "cost": 0.002998, "schemaValid": true, "pointHit": true, "boxIoU": 0.9855955978183417, "boxIoUAt50": true }, { "attempt_id": "1e417dd3c5c8c509a7b18177", "model": "google/gemini-3.1-pro-preview", "case_id": "flight-pass-ios/clean/grounding/check-in", "status": "success", "wall_time_ms": 6646.308, "cost": 0.00299, "schemaValid": true, "pointHit": true, "boxIoU": 0.9835853672612148, "boxIoUAt50": true }, { "attempt_id": "9b3642e3994916fe1cc6e67d", "model": "google/gemini-3.1-pro-preview", "case_id": "flight-pass-ios/clean/grounding/profile-tab", "status": "success", "wall_time_ms": 35165.545, "cost": 0.003014, "schemaValid": true, "pointHit": true, "boxIoU": 0.6992287115769398, "boxIoUAt50": true }, { "attempt_id": "e17c5f836508beb7855831f9", "model": "google/gemini-3.1-pro-preview", "case_id": "smart-home-android/clean/grounding/thermostat-card", "status": "success", "wall_time_ms": 4407.107, "cost": 0.00304, "schemaValid": true, "pointHit": true, "boxIoU": 0.9852107785380625, "boxIoUAt50": true }, { "attempt_id": "01f5572c48e574d29f3b32de", "model": "google/gemini-3.1-pro-preview", "case_id": "smart-home-android/clean/grounding/add-device", "status": "success", "wall_time_ms": 3347.916, "cost": 0.003014, "schemaValid": true, "pointHit": true, "boxIoU": 0.9772036136848217, "boxIoUAt50": true }, { "attempt_id": "1a38fa5e5168631d9a5f2722", "model": "google/gemini-3.1-pro-preview", "case_id": "smart-home-android/clean/grounding/more", "status": "success", "wall_time_ms": 2890.414, "cost": 0.002992, "schemaValid": true, "pointHit": true, "boxIoU": 0.33786727253989457, "boxIoUAt50": false }, { "attempt_id": "93d5198f3a8e4e15aa712c27", "model": "google/gemini-3.1-pro-preview", "case_id": "smart-home-android/clean/grounding/room-search", "status": "success", "wall_time_ms": 20549.805, "cost": 0.003002, "schemaValid": true, "pointHit": true, "boxIoU": 0.8394664765948328, "boxIoUAt50": true } ] } } }