{
  "astra": {
    "id": "astra-26",
    "name": "Astra-26",
    "repoUrl": "https://github.com/vaniagarwal343/astra-26",
    "dataPolicyUrl": "https://github.com/vaniagarwal343/astra-26/blob/c627564e23a2d0c414955fbd3c4c4858350b88f8/DATA_NOTE.md",
    "protocolUrl": "https://github.com/vaniagarwal343/astra-26/blob/c627564e23a2d0c414955fbd3c4c4858350b88f8/CONFIG.md",
    "resultsUrl": "https://github.com/vaniagarwal343/astra-26/blob/c627564e23a2d0c414955fbd3c4c4858350b88f8/RESULTS.md",
    "upstreamUrl": "https://huggingface.co/datasets/Forival/LiveBrowseComp",
    "updatedOn": "2026-09-08",
    "model": "gpt-6-astra",
    "modelLabel": "GPT-6 Astra",
    "reasoningEffort": "medium",
    "cliVersion": "0.153.4",
    "judge": "claude-opus-5",
    "judgeLabel": "Claude Opus 5",
    "softBudget": 30,
    "hardValve": 100,
    "noiseItems": 3,
    "confidenceInterval": "95% Wilson score interval, computed from published per-item verdicts; descriptive, not a paired significance test.",
    "dataPolicy": "Question text, gold answers, aliases, and transcripts are withheld under the benchmark's canary policy. Public task IDs, upstream indices, hashes, verdicts, and run telemetry remain available.",
    "costDefinition": "Model tokens priced at OpenAI's published API rates plus search spend attributed from vendor dashboards to this run. The model ran on a ChatGPT subscription, so its API cost is an estimate, not a bill. Search spend is attributed using this run's share of the day's logged calls; Keenable was within its free allowance and built-in search had no separate bill. Cost per task divides the published total by all 26 tasks, including incorrect answers and abstentions. Judge costs are excluded.",
    "costUnit": "USD per evaluated task · model + search",
    "costPerCorrectDefinition": "Total estimated model and attributed search cost across all 26 tasks divided by the observed number of correct answers. This includes spend on incorrect answers and abstentions.",
    "modelApiRatesUsdPerMillion": {
      "input": 10.0,
      "cachedInput": 1.0,
      "output": 50.0
    },
    "costReportedOn": "2026-09-08",
    "disclosure": "Exa, Valyu, and Keenable provided API credits for the earlier LBC-30 benchmark. Model and judge calls used the author's personal subscriptions. Vendors had no input into selection, tool surface, budget, or grading."
  },
  "systems": [
    {
      "id": "codex-exa",
      "label": "Exa",
      "provider": "Exa",
      "color": "#1840ed",
      "toolSurface": "MCP search / fetch"
    },
    {
      "id": "codex-builtin",
      "label": "OpenAI built-in",
      "provider": "OpenAI",
      "color": "#222222",
      "toolSurface": "Codex built-in web search"
    },
    {
      "id": "codex-keenable",
      "label": "Keenable",
      "provider": "Keenable",
      "color": "#7661c4",
      "toolSurface": "MCP search / fetch"
    },
    {
      "id": "codex-parallel",
      "label": "Parallel",
      "provider": "Parallel",
      "color": "#c45b35",
      "toolSurface": "MCP search / fetch"
    },
    {
      "id": "codex-valyu",
      "label": "Valyu",
      "provider": "Valyu",
      "color": "#347a68",
      "toolSurface": "MCP search / fetch"
    }
  ],
  "blocks": [
    {
      "id": "block2",
      "label": "LiveBrowseComp",
      "description": "Twenty-six gated LiveBrowseComp questions selected by a fixed rule from an unused pool of 282 items.",
      "nTasks": 26,
      "budget": 30,
      "source": "LiveBrowseComp",
      "closedBook": {
        "runs": 78,
        "hits": 0,
        "candidateRuns": 90,
        "candidateHitItems": 3
      },
      "noisePoints": 0.11538461538461539,
      "notes": [
        "Results from different call budgets are never pooled.",
        "One scored run per task and arm; small score gaps should be read against the preregistered three-item noise line.",
        "Thirty candidates were selected; three failed the closed-book gate and one failed the wrapper leak scan. All 26 survivors remain in the denominator."
      ]
    }
  ],
  "scores": [
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "blindId": "sys-a",
      "n": 26,
      "correct": 11,
      "accuracy": 0.4230769230769231,
      "ci": {
        "lo": 0.2554444331365656,
        "hi": 0.6105138625340828
      },
      "medianSeconds": 157.7,
      "meanToolCalls": 25,
      "costUsd": 1.2596153846153846,
      "costUsdRange": {
        "lo": 1.2596153846153846,
        "hi": 1.2596153846153846
      },
      "modelCostUsdTotal": 28.42,
      "searchCostUsdTotal": 4.33,
      "searchListPriceUsdTotal": 7.38,
      "totalCostUsd": 32.75,
      "costPerCorrectUsd": 2.977272727272727,
      "costAssumption": "Sep 7 (Pacific day): search $5.70, contents $0.15, for 769 searches and 132 fetches that day. 572 of the searches and 75 of the fetches were this run: $4.33.",
      "searchCalls": 572,
      "fetchCalls": 78,
      "toolErrors": 3,
      "stoppedAtBudget": 17,
      "overBudget": 0,
      "groundedCorrect": 9,
      "groundedRate": 0.34615384615384615,
      "queryOrigin": 0,
      "decidedBy": {
        "normalizer": 8,
        "mustFail": 0,
        "llm": 18,
        "noResponse": 0
      }
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "blindId": "sys-b",
      "n": 26,
      "correct": 11,
      "accuracy": 0.4230769230769231,
      "ci": {
        "lo": 0.2554444331365656,
        "hi": 0.6105138625340828
      },
      "medianSeconds": 256.7,
      "meanToolCalls": 19.73076923076923,
      "costUsd": 3.8423076923076924,
      "costUsdRange": {
        "lo": 3.8423076923076924,
        "hi": 3.8423076923076924
      },
      "modelCostUsdTotal": 99.9,
      "searchCostUsdTotal": 0.0,
      "searchListPriceUsdTotal": 0.0,
      "totalCostUsd": 99.9,
      "costPerCorrectUsd": 9.081818181818182,
      "costAssumption": "no separate bill; Codex's native search is part of the subscription. $0.",
      "searchCalls": 513,
      "fetchCalls": 0,
      "toolErrors": 0,
      "stoppedAtBudget": 11,
      "overBudget": 0,
      "groundedCorrect": null,
      "groundedRate": null,
      "queryOrigin": 3,
      "decidedBy": {
        "normalizer": 7,
        "mustFail": 0,
        "llm": 19,
        "noResponse": 0
      }
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "blindId": "sys-d",
      "n": 26,
      "correct": 10,
      "accuracy": 0.38461538461538464,
      "ci": {
        "lo": 0.22428594435712798,
        "hi": 0.5746514991488447
      },
      "medianSeconds": 170.9,
      "meanToolCalls": 26.346153846153847,
      "costUsd": 1.3038461538461539,
      "costUsdRange": {
        "lo": 1.3038461538461539,
        "hi": 1.3038461538461539
      },
      "modelCostUsdTotal": 33.9,
      "searchCostUsdTotal": 0.0,
      "searchListPriceUsdTotal": 2.74,
      "totalCostUsd": 33.9,
      "costPerCorrectUsd": 3.3899999999999997,
      "costAssumption": "685 requests inside the monthly free allowance of 100,000; $4 per 1k after that. $0 billed; $2.74 at list.",
      "searchCalls": 610,
      "fetchCalls": 75,
      "toolErrors": 4,
      "stoppedAtBudget": 20,
      "overBudget": 0,
      "groundedCorrect": 8,
      "groundedRate": 0.3076923076923077,
      "queryOrigin": 0,
      "decidedBy": {
        "normalizer": 5,
        "mustFail": 0,
        "llm": 21,
        "noResponse": 0
      }
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "blindId": "sys-e",
      "n": 26,
      "correct": 7,
      "accuracy": 0.2692307692307692,
      "ci": {
        "lo": 0.13704448322018375,
        "hi": 0.4608304037917617
      },
      "medianSeconds": 214.8,
      "meanToolCalls": 26.96153846153846,
      "costUsd": 1.3665384615384615,
      "costUsdRange": {
        "lo": 1.3665384615384615,
        "hi": 1.3665384615384615
      },
      "modelCostUsdTotal": 32.14,
      "searchCostUsdTotal": 3.39,
      "searchListPriceUsdTotal": 3.19,
      "totalCostUsd": 35.53,
      "costPerCorrectUsd": 5.075714285714286,
      "costAssumption": "Sep 8 (UTC day): search $2.82, extract $0.18, for 529 searches and 90 extracts; the run's first 22 minutes fall in the Sep 7 bucket. the Sep 8 rates ($5.33 per 1k searches, $2 per 1k extracts) applied to the run's 596 searches and 105 extracts: $3.39.",
      "searchCalls": 596,
      "fetchCalls": 105,
      "toolErrors": 10,
      "stoppedAtBudget": 19,
      "overBudget": 0,
      "groundedCorrect": 4,
      "groundedRate": 0.15384615384615385,
      "queryOrigin": 0,
      "decidedBy": {
        "normalizer": 3,
        "mustFail": 0,
        "llm": 23,
        "noResponse": 0
      }
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "blindId": "sys-c",
      "n": 26,
      "correct": 6,
      "accuracy": 0.23076923076923078,
      "ci": {
        "lo": 0.11033849405783291,
        "hi": 0.42051554078943687
      },
      "medianSeconds": 299.2,
      "meanToolCalls": 28.807692307692307,
      "costUsd": 1.5557692307692308,
      "costUsdRange": {
        "lo": 1.5557692307692308,
        "hi": 1.5557692307692308
      },
      "modelCostUsdTotal": 36.83,
      "searchCostUsdTotal": 3.62,
      "searchListPriceUsdTotal": 5.31,
      "totalCostUsd": 40.45,
      "costPerCorrectUsd": 6.741666666666667,
      "costAssumption": "Sep 7: search $4.84, contents $0.10, for 854 searches and 174 fetches that day. 630 of the searches and 95 of the fetches were this run: $3.62.",
      "searchCalls": 630,
      "fetchCalls": 119,
      "toolErrors": 24,
      "stoppedAtBudget": 21,
      "overBudget": 0,
      "groundedCorrect": 4,
      "groundedRate": 0.15384615384615385,
      "queryOrigin": 0,
      "decidedBy": {
        "normalizer": 2,
        "mustFail": 0,
        "llm": 24,
        "noResponse": 0
      }
    }
  ],
  "tasks": [
    {
      "id": "B01",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 6,
      "sha256": "74b1ece51bd00c4528f284a90d4956eb9eaa575b73ff83cf64a71a0f1d28bc27",
      "correctSystems": 4,
      "contraryEvidence": false
    },
    {
      "id": "B02",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 17,
      "sha256": "464f2d145ac8f25f6f3a5927bb8bcd5ab0bcefea23d7e2f8435f5851aef54447",
      "correctSystems": 0,
      "contraryEvidence": false
    },
    {
      "id": "B03",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 28,
      "sha256": "5acc6d40e6aeb9804783a8d817c85bb1043832fcd995a05784072342c8bf6daa",
      "correctSystems": 2,
      "contraryEvidence": false
    },
    {
      "id": "B04",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 38,
      "sha256": "1f2e908316a2b7d8fcd5f460c83ea78e951a86adb91ab0a2f2b66444c7962587",
      "correctSystems": 2,
      "contraryEvidence": false
    },
    {
      "id": "B05",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 50,
      "sha256": "f88594740a2352e0d51dc125da7763fcdac6f24ca6d8a79e6e8c7bbdb28ee073",
      "correctSystems": 0,
      "contraryEvidence": false
    },
    {
      "id": "B06",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 60,
      "sha256": "c69eb013b9c4a3177c72169821026d2a66ae6f635bd0c95830f862e7ed1d89ac",
      "correctSystems": 3,
      "contraryEvidence": false
    },
    {
      "id": "B07",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 70,
      "sha256": "34fc187de0052e001567715e55d1acdf3b7a3226ef7389fb7c629b356c89a37f",
      "correctSystems": 0,
      "contraryEvidence": true
    },
    {
      "id": "B08",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 81,
      "sha256": "018781d6ffdf491b6081014d41fffdbee194cdf21ecb26afb54297422b1e5b21",
      "correctSystems": 5,
      "contraryEvidence": false
    },
    {
      "id": "B09",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 93,
      "sha256": "1b498d0f7a4773e093af38923e600c7f1d87e03180b794211fc3fb094fe0ad51",
      "correctSystems": 5,
      "contraryEvidence": false
    },
    {
      "id": "B10",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 103,
      "sha256": "3a069aeecef46c4c7e663aa141c96102d5f4172e0e00a10d83d17146eb141087",
      "correctSystems": 1,
      "contraryEvidence": false
    },
    {
      "id": "B11",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 114,
      "sha256": "463386df1a91a4a342ed4f7b49b5dffdb001f01676ec8e780d617d2a63cc0065",
      "correctSystems": 1,
      "contraryEvidence": false
    },
    {
      "id": "B13",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "DATED",
      "upstreamIdx": 135,
      "sha256": "0281e063935c4f4cfd788ab0942f7101b0c9f16af1700d737ced3704dc671172",
      "correctSystems": 0,
      "contraryEvidence": false
    },
    {
      "id": "B14",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "DATED",
      "upstreamIdx": 145,
      "sha256": "2aa59bc90f9c526a7360de2e58df9a8b0fb63d829bd5fc7bd6128b036d53cc1f",
      "correctSystems": 5,
      "contraryEvidence": false
    },
    {
      "id": "B15",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 159,
      "sha256": "e9a3f4b6055e401c5046869aa59f9633ad7c23615864474cc7014a49b36fa63c",
      "correctSystems": 2,
      "contraryEvidence": false
    },
    {
      "id": "B16",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 168,
      "sha256": "3987009e03c6d97073f202a9eb07cc1362782fd2739c55f95d6ef19e961487e8",
      "correctSystems": 0,
      "contraryEvidence": false
    },
    {
      "id": "B17",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 177,
      "sha256": "8b6360c39fee164f87cf56a96f25adcc794ded807bd5b194640e304e8f8ae170",
      "correctSystems": 4,
      "contraryEvidence": false
    },
    {
      "id": "B18",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 188,
      "sha256": "7e347f962055943a25f6789dfd27fb44c95a373da66c7db93a4ca1252a14153a",
      "correctSystems": 0,
      "contraryEvidence": false
    },
    {
      "id": "B19",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 200,
      "sha256": "e4ef050db922c45424af7c5be8c65b9e1fa5b89c9b550938de317f6ff7743301",
      "correctSystems": 2,
      "contraryEvidence": false
    },
    {
      "id": "B21",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 222,
      "sha256": "a4c7988359c9825723d6e9097065e022df8963de9ce11becdfc45013c4c8549c",
      "correctSystems": 0,
      "contraryEvidence": true
    },
    {
      "id": "B23",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 242,
      "sha256": "ba11f8ae14fb54de83947e3ab0fe7d855f2d0913b278e566ac5d5261aaa05b3c",
      "correctSystems": 1,
      "contraryEvidence": false
    },
    {
      "id": "B25",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 263,
      "sha256": "abc8cd90ec4a9a00799f78e4d537903a713025c1799cc12fb5711b45e0944e09",
      "correctSystems": 1,
      "contraryEvidence": false
    },
    {
      "id": "B26",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 274,
      "sha256": "d11d805f1341fed77310765c035559bcf3470559ee5962e82aab2b564e4c4266",
      "correctSystems": 0,
      "contraryEvidence": true
    },
    {
      "id": "B27",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 285,
      "sha256": "df34d83575b56d3df9fd9e7010db6f644565ec3d0857448682de0366c90ad36b",
      "correctSystems": 4,
      "contraryEvidence": false
    },
    {
      "id": "B28",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "DATED",
      "upstreamIdx": 295,
      "sha256": "73de2633dbacd99d90e0883fc4c9660027bf718f948416430877fe4a0bf5ad4f",
      "correctSystems": 0,
      "contraryEvidence": true
    },
    {
      "id": "B29",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 304,
      "sha256": "85c679946201575bd085a7a7d1def76e68c5df108e4cfa0cfd6cfcc9e4fc98a7",
      "correctSystems": 3,
      "contraryEvidence": false
    },
    {
      "id": "B30",
      "blockId": "block2",
      "source": "lbc",
      "sourceLabel": "LiveBrowseComp",
      "stratum": "UNDATED",
      "upstreamIdx": 314,
      "sha256": "9c7452077e2f9c9f316d35713772cc4d9665b5c04d580a0fed3446f8cd6701dd",
      "correctSystems": 0,
      "contraryEvidence": true
    }
  ],
  "runs": [
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B01",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 14,
      "fetchCalls": 4,
      "toolCalls": 18,
      "seconds": 73.5,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B02",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 10,
      "fetchCalls": 4,
      "toolCalls": 14,
      "seconds": 107.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B03",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 126.6,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B04",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 19,
      "fetchCalls": 7,
      "toolCalls": 26,
      "seconds": 107.3,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B05",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 26,
      "fetchCalls": 4,
      "toolCalls": 30,
      "seconds": 142.4,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B06",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 26,
      "fetchCalls": 4,
      "toolCalls": 30,
      "seconds": 187.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B07",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 5,
      "fetchCalls": 3,
      "toolCalls": 8,
      "seconds": 48.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B08",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 24,
      "fetchCalls": 6,
      "toolCalls": 30,
      "seconds": 175.5,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B09",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 13,
      "fetchCalls": 3,
      "toolCalls": 16,
      "seconds": 121.1,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B10",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 20,
      "fetchCalls": 6,
      "toolCalls": 26,
      "seconds": 128.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B11",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 337.8,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B13",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 229.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B14",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 6,
      "fetchCalls": 4,
      "toolCalls": 10,
      "seconds": 102.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B15",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 26,
      "fetchCalls": 4,
      "toolCalls": 30,
      "seconds": 173.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B16",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 137.6,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B17",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 25,
      "fetchCalls": 5,
      "toolCalls": 30,
      "seconds": 295.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B18",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 225.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B19",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 261.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B21",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 123.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B23",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 352.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B25",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 351.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B26",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 331.7,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B27",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 7,
      "fetchCalls": 6,
      "toolCalls": 13,
      "seconds": 123.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B28",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 220.6,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B29",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 5,
      "fetchCalls": 4,
      "toolCalls": 9,
      "seconds": 66.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-exa",
      "taskId": "B30",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 175.7,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B01",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 8,
      "fetchCalls": 0,
      "toolCalls": 8,
      "seconds": 81.3,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B02",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 8,
      "fetchCalls": 0,
      "toolCalls": 8,
      "seconds": 50.7,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B03",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 335.3,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B04",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 10,
      "fetchCalls": 0,
      "toolCalls": 10,
      "seconds": 96.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B05",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 286.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B06",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 23,
      "fetchCalls": 0,
      "toolCalls": 23,
      "seconds": 227.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": true,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B07",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 20,
      "fetchCalls": 0,
      "toolCalls": 20,
      "seconds": 191.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B08",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 7,
      "fetchCalls": 0,
      "toolCalls": 7,
      "seconds": 94.3,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B09",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 6,
      "fetchCalls": 0,
      "toolCalls": 6,
      "seconds": 87.9,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B10",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 18,
      "fetchCalls": 0,
      "toolCalls": 18,
      "seconds": 178.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": true,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B11",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 606.9,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B13",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 551.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B14",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 673.9,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B15",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 11,
      "fetchCalls": 0,
      "toolCalls": 11,
      "seconds": 89.7,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B16",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 307.5,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B17",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 20,
      "fetchCalls": 0,
      "toolCalls": 20,
      "seconds": 376.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B18",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 11,
      "fetchCalls": 0,
      "toolCalls": 11,
      "seconds": 205.7,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B19",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 24,
      "fetchCalls": 0,
      "toolCalls": 24,
      "seconds": 322.8,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B21",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 348.8,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B23",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 499.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B25",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 10,
      "fetchCalls": 0,
      "toolCalls": 10,
      "seconds": 191.8,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B26",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 508.9,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B27",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 4,
      "fetchCalls": 0,
      "toolCalls": 4,
      "seconds": 95.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B28",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 478.8,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B29",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 3,
      "fetchCalls": 0,
      "toolCalls": 3,
      "seconds": 54.7,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": true,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-builtin",
      "taskId": "B30",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 431.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": null,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B01",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 15,
      "fetchCalls": 3,
      "toolCalls": 18,
      "seconds": 83.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B02",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 5,
      "fetchCalls": 4,
      "toolCalls": 9,
      "seconds": 76.6,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B03",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 130.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B04",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 24,
      "fetchCalls": 6,
      "toolCalls": 30,
      "seconds": 150.3,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B05",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 202.8,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B06",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 185.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B07",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 168.6,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B08",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 24,
      "fetchCalls": 6,
      "toolCalls": 30,
      "seconds": 159.0,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B09",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 8,
      "fetchCalls": 5,
      "toolCalls": 13,
      "seconds": 98.9,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B10",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 21,
      "fetchCalls": 9,
      "toolCalls": 30,
      "seconds": 133.1,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B11",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 320.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B13",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 24,
      "fetchCalls": 6,
      "toolCalls": 30,
      "seconds": 249.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B14",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 10,
      "fetchCalls": 4,
      "toolCalls": 14,
      "seconds": 123.6,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B15",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 27,
      "fetchCalls": 3,
      "toolCalls": 30,
      "seconds": 411.6,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B16",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 170.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B17",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 21,
      "fetchCalls": 9,
      "toolCalls": 30,
      "seconds": 205.1,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B18",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 27,
      "fetchCalls": 3,
      "toolCalls": 30,
      "seconds": 218.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B19",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 27,
      "fetchCalls": 3,
      "toolCalls": 30,
      "seconds": 651.8,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B21",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 152.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B23",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 225.3,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B25",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 306.9,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B26",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 383.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B27",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 16,
      "fetchCalls": 5,
      "toolCalls": 21,
      "seconds": 160.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B28",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 171.7,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B29",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 6,
      "fetchCalls": 4,
      "toolCalls": 10,
      "seconds": 77.3,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-keenable",
      "taskId": "B30",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 197.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B01",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 12,
      "fetchCalls": 6,
      "toolCalls": 18,
      "seconds": 119.7,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B02",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 6,
      "fetchCalls": 5,
      "toolCalls": 11,
      "seconds": 86.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B03",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 130.8,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B04",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 23,
      "fetchCalls": 7,
      "toolCalls": 30,
      "seconds": 129.3,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B05",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 21,
      "fetchCalls": 9,
      "toolCalls": 30,
      "seconds": 240.5,
      "attempts": 1,
      "toolErrors": 4,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B06",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 226.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B07",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 120.5,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B08",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 16,
      "fetchCalls": 7,
      "toolCalls": 23,
      "seconds": 135.5,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B09",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 12,
      "fetchCalls": 6,
      "toolCalls": 18,
      "seconds": 213.9,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B10",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 16,
      "fetchCalls": 8,
      "toolCalls": 24,
      "seconds": 101.5,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B11",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 349.8,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B13",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 26,
      "fetchCalls": 4,
      "toolCalls": 30,
      "seconds": 339.6,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B14",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 10,
      "fetchCalls": 11,
      "toolCalls": 21,
      "seconds": 189.6,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B15",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 380.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B16",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 26,
      "fetchCalls": 4,
      "toolCalls": 30,
      "seconds": 183.9,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B17",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 22,
      "fetchCalls": 8,
      "toolCalls": 30,
      "seconds": 230.5,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B18",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 282.7,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B19",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 252.5,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B21",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 120.7,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B23",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 27,
      "fetchCalls": 3,
      "toolCalls": 30,
      "seconds": 254.5,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B25",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 405.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B26",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 27,
      "fetchCalls": 3,
      "toolCalls": 30,
      "seconds": 509.5,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B27",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 8,
      "fetchCalls": 8,
      "toolCalls": 16,
      "seconds": 157.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B28",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 25,
      "fetchCalls": 5,
      "toolCalls": 30,
      "seconds": 202.9,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B29",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 304.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-parallel",
      "taskId": "B30",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 215.7,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B01",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 18,
      "fetchCalls": 11,
      "toolCalls": 29,
      "seconds": 245.2,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B02",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 18,
      "fetchCalls": 11,
      "toolCalls": 29,
      "seconds": 255.3,
      "attempts": 1,
      "toolErrors": 2,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B03",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 216.5,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B04",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 25,
      "fetchCalls": 5,
      "toolCalls": 30,
      "seconds": 278.5,
      "attempts": 1,
      "toolErrors": 2,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B05",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 24,
      "fetchCalls": 6,
      "toolCalls": 30,
      "seconds": 253.9,
      "attempts": 1,
      "toolErrors": 5,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B06",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 26,
      "fetchCalls": 4,
      "toolCalls": 30,
      "seconds": 309.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B07",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 373.9,
      "attempts": 1,
      "toolErrors": 2,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B08",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 23,
      "fetchCalls": 7,
      "toolCalls": 30,
      "seconds": 249.8,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B09",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 20,
      "fetchCalls": 8,
      "toolCalls": 28,
      "seconds": 215.7,
      "attempts": 1,
      "toolErrors": 2,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B10",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 19,
      "fetchCalls": 11,
      "toolCalls": 30,
      "seconds": 237.5,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B11",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 376.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B13",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 25,
      "fetchCalls": 5,
      "toolCalls": 30,
      "seconds": 320.3,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B14",
      "correct": true,
      "decidedBy": "normalizer",
      "searchCalls": 12,
      "fetchCalls": 8,
      "toolCalls": 20,
      "seconds": 230.9,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": true,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B15",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 25,
      "fetchCalls": 5,
      "toolCalls": 30,
      "seconds": 405.9,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B16",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 27,
      "fetchCalls": 3,
      "toolCalls": 30,
      "seconds": 211.6,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B17",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 24,
      "fetchCalls": 6,
      "toolCalls": 30,
      "seconds": 618.0,
      "attempts": 1,
      "toolErrors": 3,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B18",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 26,
      "fetchCalls": 4,
      "toolCalls": 30,
      "seconds": 314.6,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B19",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 25,
      "fetchCalls": 5,
      "toolCalls": 30,
      "seconds": 552.1,
      "attempts": 1,
      "toolErrors": 3,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B21",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 236.6,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B23",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 382.1,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B25",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 29,
      "fetchCalls": 1,
      "toolCalls": 30,
      "seconds": 566.3,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B26",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 28,
      "fetchCalls": 2,
      "toolCalls": 30,
      "seconds": 491.4,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B27",
      "correct": true,
      "decidedBy": "llm",
      "searchCalls": 9,
      "fetchCalls": 4,
      "toolCalls": 13,
      "seconds": 162.2,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": false
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B28",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 26,
      "fetchCalls": 4,
      "toolCalls": 30,
      "seconds": 294.2,
      "attempts": 1,
      "toolErrors": 1,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B29",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 27,
      "fetchCalls": 3,
      "toolCalls": 30,
      "seconds": 395.0,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    },
    {
      "blockId": "block2",
      "systemId": "codex-valyu",
      "taskId": "B30",
      "correct": false,
      "decidedBy": "llm",
      "searchCalls": 30,
      "fetchCalls": 0,
      "toolCalls": 30,
      "seconds": 304.2,
      "attempts": 1,
      "toolErrors": 0,
      "groundedCorrect": false,
      "queryOrigin": false,
      "stoppedAtBudget": true
    }
  ],
  "exclusions": [
    {
      "id": "B12",
      "upstreamIdx": 124,
      "gate": "wrapper-leak(exa)"
    },
    {
      "id": "B20",
      "upstreamIdx": 211,
      "gate": "closed-book"
    },
    {
      "id": "B22",
      "upstreamIdx": 233,
      "gate": "closed-book"
    },
    {
      "id": "B24",
      "upstreamIdx": 253,
      "gate": "closed-book"
    }
  ],
  "sources": {
    "repository": "https://github.com/vaniagarwal343/astra-26",
    "sourceCommit": "c627564e23a2d0c414955fbd3c4c4858350b88f8",
    "sourceRef": "c627564e23a2d0c414955fbd3c4c4858350b88f8",
    "sourceState": "committed",
    "note": "Source-file hashes identify this public snapshot. When sourceState is working-tree, sourceCommit records the repository's base commit; the exported publication updates are not yet represented by that commit. Artifact links use main until those updates have a matching commit.",
    "files": [
      {
        "path": "configs/protocol.json",
        "sha256": "f36a05278f42c6afa1b93a065d80a1448a8b85d6ae40176bb4d64173aed27121"
      },
      {
        "path": "configs/judge.json",
        "sha256": "dc03327f3b4f24bc4e50a19970c454f03ddd6e50a9f7d3c276537e7062856cc2"
      },
      {
        "path": "configs/codex_exa.json",
        "sha256": "cbe4c2f3396cef30ea7a14b1b489839cf020de3e1811c45c5e2a985e9639a03f"
      },
      {
        "path": "RESULTS.md",
        "sha256": "db84945f95dc2e26b98b10fa5187c23aa53837a4e1c83e580724f71a51f7825b"
      },
      {
        "path": "ANALYSIS.md",
        "sha256": "d15b2f003ca728d2ab56a20bea820bf8ae28dff39f007b5915d41205ee60e4ca"
      },
      {
        "path": "CONFIG.md",
        "sha256": "657a41e188f0717b6bd02fab1502c86f7b31900b60dcf14b5e6fb6499faddefa"
      },
      {
        "path": "DATA_NOTE.md",
        "sha256": "ff41ec2f52630472a96c7985286d676d91199f1254dbefe21acf5be411504c4b"
      },
      {
        "path": "tasks/hashes.json",
        "sha256": "25f67c9fcf24ae6425b9e5fb9eae719afd0f258cc6d2bee94b8faa407114270b"
      },
      {
        "path": "published/per_run.json",
        "sha256": "cbf1eb709aa61637a6d88616ed4d5c31b077df2f910165a01be1648ddd70a902"
      },
      {
        "path": "published/grounding_summary.json",
        "sha256": "601c2940a65bdfd2dc7a9e4f005d0e374e31abfcd264f1542c226c4a71b01d80"
      },
      {
        "path": "published/blinding_map.json",
        "sha256": "c9d18292949537091b8f3e7e1f86cf0b8bdcf321bfbda91dfabfce39bf1ffa87"
      },
      {
        "path": "published/gate_closed_book.json",
        "sha256": "c2d84636efc3dbf19c69843ec5de9d6adcd3dd49cb74ecf451351b98d9ce6983"
      },
      {
        "path": "published/contrary_evidence.json",
        "sha256": "4da30410efc4361f8452519ade5f052ff1dab1fbaa3ebf3a5ee1166026884b48"
      },
      {
        "path": "published/judging_public/codex-exa.json",
        "sha256": "977547930284c3a8175fb91ef18b93d0ba69bd64a243da53109bed959118b772"
      },
      {
        "path": "published/judging_public/codex-builtin.json",
        "sha256": "737e58c97fc71417b9a7b7edd10737c57c312b36d4ac11ca616dbbc04867c0f2"
      },
      {
        "path": "published/judging_public/codex-keenable.json",
        "sha256": "5cebc1aff40619d1e2ebb81cb5bf2a9882d70cd766410d7e615365c58d47261d"
      },
      {
        "path": "published/judging_public/codex-parallel.json",
        "sha256": "53fdb738d85bb74555938ba7c55988c9aae3ed06077480ffa24535325e3d365c"
      },
      {
        "path": "published/judging_public/codex-valyu.json",
        "sha256": "5a16af8c19eaed3c3998a57d9738e1efd721b1298f951df4be8be1a2e084edc2"
      }
    ]
  }
}