{
 "study": {
  "id": "harness-bench",
  "round": 1,
  "model": "claude-fable-5-2",
  "effort": "high",
  "tasks": 12,
  "systems": 4,
  "runs": 48,
  "judgments": 1280,
  "criteriaPerSystem": 320,
  "selectedOn": "2026-11-09",
  "firstRunStarted": "2026-11-10T00:32:35Z",
  "lastRunEnded": "2026-11-10T06:07:36Z",
  "freezeTag": "freeze-r1",
  "freezeCommit": "b48847bdf9cad36bbb132bc47a10b728e1fa8dd1",
  "resultsTag": "freeze-r1-a3",
  "resultsCommit": "c959b776c069b4ff06acea9ceea26c9f1c8ad8a7",
  "configSha256": "75c2ecd8a431a76eda2d50c184430f945e17cd51a815599a6566e01536449b08",
  "manifestSha256": "adf8e44e505f1625aa019423cfba9fa04d802c05e8570434607e1e07563eb516",
  "resultsSha256": "c2b752c712233ff16ec0a8a5820b18c63c4e7f22005e78cc873b1bc0cf73e469",
  "bootstrapDraws": 10000
 },
 "dataset": {
  "repo": "ScaleAI/researchrubrics",
  "revision": "85de3115053d1453ed612caacf4a405edc1ad756",
  "questionsUpstream": 101,
  "mainOriginal": 15,
  "smoke": 2,
  "droppedFromMain": [
   "rr-605371",
   "rr-6053bb",
   "rr-9af31b"
  ],
  "errorCriteria": 35,
  "citationDependentCriteria": 31
 },
 "pricingUsdPerMillionTokens": {
  "input": 8.0,
  "output": 40.0,
  "cacheWrite": 10.0,
  "cacheRead": 0.2
 },
 "tools": {
  "backend": "exa",
  "searchType": "auto",
  "searchMaxResults": 5,
  "snippetChars": 500,
  "fetchMaxChars": 30000
 },
 "outputAllowance": {
  "closed_book": 64000,
  "plain_tools": 64000,
  "gpt_researcher": 20000,
  "deepagents": 20000
 },
 "safeguards": {
  "maxWallSeconds": 14400,
  "maxModelCalls": 3000,
  "maxResearchCalls": 5000
 },
 "judges": {
  "jev": {
   "model": "jev-1.14.0",
   "attempts": 1280,
   "verdicts": 1280,
   "costUsd": 0.3503,
   "longContextJudgments": 0
  },
  "astra": {
   "model": "gpt-6-astra",
   "cli": "codex-cli 0.162.1",
   "reasoningEffort": "medium",
   "mode": "single",
   "sessions": 1283,
   "attempts": 1283,
   "verdicts": 1280,
   "errorAttempts": {
    "error_quote_mismatch": 3
   }
  },
  "labelPairs": {
   "cannot_judge|cannot_judge": 4,
   "cannot_judge|met": 1,
   "met|met": 558,
   "met|not_met": 44,
   "not_met|cannot_judge": 3,
   "not_met|met": 87,
   "not_met|not_met": 583
  },
  "disagreements": 135,
  "unresolved": 139
 },
 "systems": {
  "closed_book": {
   "runs": {
    "scheduled": 12,
    "completed": 12,
    "withReport": 12,
    "retries": 0,
    "policyViolations": 0
   },
   "scores": {
    "total": {
     "jev": 0.5572207164428726,
     "astra": 0.558935250715603,
     "two_judge_mean": 0.5580779835792378,
     "consensus_lo": 0.4977292670217695,
     "consensus_hi": 0.6090134263862924
    },
    "adapted_no_citation": {
     "astra": 0.5768624041689447,
     "jev": 0.5748698244392828,
     "two_judge_mean": 0.5758661143041137
    },
    "adapted_citation_dependent": {
     "astra": 0.3908836261777439,
     "jev": 0.48179271708683474,
     "two_judge_mean": 0.43633817163228933
    },
    "adapted_content": {
     "astra": 0.44941059757236224,
     "jev": 0.43115429505135394,
     "two_judge_mean": 0.4402824463118581
    },
    "adapted_coverage": {
     "astra": 0.6345738667258054,
     "jev": 0.6343819023070141,
     "two_judge_mean": 0.6344778845164097
    },
    "adapted_presentation": {
     "astra": 0.6130379424497071,
     "jev": 0.6433409727527375,
     "two_judge_mean": 0.6281894576012222
    },
    "adapted_citation": {
     "astra": 0.0,
     "jev": 0.0,
     "two_judge_mean": 0.0
    }
   },
   "errorsPresentMean": {
    "astra": 0.75,
    "jev": 0.5
   },
   "judgments": {
    "expected": 320,
    "disagree": 33,
    "cannotJudge": 2,
    "unresolvedError": 0,
    "unresolved": 34,
    "resolvedByAiPass": 7
   },
   "cost": {
    "modelTotal": 5.4575,
    "modelMean": 0.45479066666666673,
    "modelMedian": 0.41000000000000003,
    "modelMax": 0.834072,
    "retrievalTotal": 0.0,
    "retrievalMean": 0.0,
    "generationTotal": 5.457488,
    "generationMean": 0.4547906666666666,
    "generationMedian": 0.41000000000000003,
    "generationMin": 0.20396,
    "generationMax": 0.834072,
    "meanByTokenKind": {
     "input": 0.002754,
     "cacheWrite": 0.0,
     "cacheRead": 0.0,
     "output": 0.45203666666666664
    }
   },
   "time": {
    "meanSeconds": 122.11583333333333,
    "medianSeconds": 109.1,
    "minSeconds": 53.7,
    "maxSeconds": 207.86
   },
   "activity": {
    "modelCallsMean": 1.0,
    "researchCallsMean": 0.0,
    "searchCallsMean": 0.0,
    "fetchCallsMean": 0.0,
    "reportCharsMean": 13497.166666666666,
    "outputTokensMean": 11300.916666666666,
    "inputTokensMean": 344.25,
    "tokensMean": {
     "input": 344.25,
     "cacheWrite": 0.0,
     "cacheRead": 0.0,
     "output": 11300.916666666666
    },
    "modelCallsTotal": 12,
    "outputLimitStops": 0,
    "runsWithOutputLimitStop": 0,
    "refusalStops": 0,
    "runsWithRefusalStop": 0
   }
  },
  "plain_tools": {
   "runs": {
    "scheduled": 12,
    "completed": 12,
    "withReport": 12,
    "retries": 0,
    "policyViolations": 0
   },
   "scores": {
    "total": {
     "jev": 0.5198438625857492,
     "astra": 0.5161767574464347,
     "two_judge_mean": 0.5180103100160921,
     "consensus_lo": 0.4588316130701298,
     "consensus_hi": 0.581119824572117
    },
    "adapted_no_citation": {
     "astra": 0.5192500251924375,
     "jev": 0.5203446212677673,
     "two_judge_mean": 0.5197973232301024
    },
    "adapted_citation_dependent": {
     "astra": 0.5742169595110771,
     "jev": 0.5742169595110771,
     "two_judge_mean": 0.5742169595110771
    },
    "adapted_content": {
     "astra": 0.4399824929971989,
     "jev": 0.44697595704948645,
     "two_judge_mean": 0.4434792250233427
    },
    "adapted_coverage": {
     "astra": 0.585856954256261,
     "jev": 0.579382252184912,
     "two_judge_mean": 0.5826196032205866
    },
    "adapted_presentation": {
     "astra": 0.5757830404889228,
     "jev": 0.5820218996689585,
     "two_judge_mean": 0.5789024700789406
    },
    "adapted_citation": {
     "astra": 0.48,
     "jev": 0.27999999999999997,
     "two_judge_mean": 0.38
    }
   },
   "errorsPresentMean": {
    "astra": 0.75,
    "jev": 0.5
   },
   "judgments": {
    "expected": 320,
    "disagree": 32,
    "cannotJudge": 2,
    "unresolvedError": 0,
    "unresolved": 33,
    "resolvedByAiPass": 12
   },
   "cost": {
    "modelTotal": 32.7875,
    "modelMean": 2.732291666666667,
    "modelMedian": 2.6751525000000003,
    "modelMax": 5.044466,
    "retrievalTotal": 2.33,
    "retrievalMean": 0.19416666666666668,
    "generationTotal": 35.1175,
    "generationMean": 2.926458333333333,
    "generationMedian": 2.87537,
    "generationMin": 1.540552,
    "generationMax": 5.361466,
    "meanByTokenKind": {
     "input": 0.00017866666666666668,
     "cacheWrite": 1.6585816666666666,
     "cacheRead": 0.15744805,
     "output": 0.9160833333333334
    }
   },
   "time": {
    "meanSeconds": 308.3183333333333,
    "medianSeconds": 280.94500000000005,
    "minSeconds": 112.43,
    "maxSeconds": 527.44
   },
   "activity": {
    "modelCallsMean": 10.166666666666666,
    "researchCallsMean": 49.0,
    "searchCallsMean": 24.25,
    "fetchCallsMean": 24.75,
    "reportCharsMean": 15236.5,
    "outputTokensMean": 22902.083333333332,
    "inputTokensMean": 953120.75,
    "tokensMean": {
     "input": 22.333333333333332,
     "cacheWrite": 165858.16666666666,
     "cacheRead": 787240.25,
     "output": 22902.083333333332
    },
    "modelCallsTotal": 122,
    "outputLimitStops": 0,
    "runsWithOutputLimitStop": 0,
    "refusalStops": 0,
    "runsWithRefusalStop": 0
   }
  },
  "gpt_researcher": {
   "runs": {
    "scheduled": 12,
    "completed": 12,
    "withReport": 12,
    "retries": 0,
    "policyViolations": 0
   },
   "scores": {
    "total": {
     "jev": 0.4227351150605179,
     "astra": 0.443526520747878,
     "two_judge_mean": 0.433130817904198,
     "consensus_lo": 0.35307072512758597,
     "consensus_hi": 0.5066145904252444
    },
    "adapted_no_citation": {
     "astra": 0.45130801452494396,
     "jev": 0.41905480233335285,
     "two_judge_mean": 0.4351814084291484
    },
    "adapted_citation_dependent": {
     "astra": 0.4092818945760122,
     "jev": 0.4136108989050165,
     "two_judge_mean": 0.41144639674051436
    },
    "adapted_content": {
     "astra": 0.3439204014939309,
     "jev": 0.2197233893557423,
     "two_judge_mean": 0.2818218954248366
    },
    "adapted_coverage": {
     "astra": 0.5133964131889043,
     "jev": 0.48852253233401927,
     "two_judge_mean": 0.5009594727614618
    },
    "adapted_presentation": {
     "astra": 0.6354978354978355,
     "jev": 0.4264323911382735,
     "two_judge_mean": 0.5309651133180545
    },
    "adapted_citation": {
     "astra": 0.4,
     "jev": 0.2,
     "two_judge_mean": 0.3
    }
   },
   "errorsPresentMean": {
    "astra": 0.9166666666666666,
    "jev": 0.5833333333333334
   },
   "judgments": {
    "expected": 320,
    "disagree": 43,
    "cannotJudge": 3,
    "unresolvedError": 0,
    "unresolved": 44,
    "resolvedByAiPass": 9
   },
   "cost": {
    "modelTotal": 87.0201,
    "modelMean": 7.251678000000001,
    "modelMedian": 6.891772,
    "modelMax": 13.331976,
    "retrievalTotal": 6.576,
    "retrievalMean": 0.5479999999999999,
    "generationTotal": 93.596136,
    "generationMean": 7.799678,
    "generationMedian": 7.443772,
    "generationMin": 5.792984000000001,
    "generationMax": 13.835975999999999,
    "meanByTokenKind": {
     "input": 4.967928,
     "cacheWrite": 0.0,
     "cacheRead": 0.0,
     "output": 2.28375
    }
   },
   "time": {
    "meanSeconds": 444.885,
    "medianSeconds": 429.99,
    "minSeconds": 414.99,
    "maxSeconds": 547.01
   },
   "activity": {
    "modelCallsMean": 36.083333333333336,
    "researchCallsMean": 274.0,
    "searchCallsMean": 45.666666666666664,
    "fetchCallsMean": 228.33333333333334,
    "reportCharsMean": 26069.75,
    "outputTokensMean": 57093.75,
    "inputTokensMean": 620991.0,
    "tokensMean": {
     "input": 620991.0,
     "cacheWrite": 0.0,
     "cacheRead": 0.0,
     "output": 57093.75
    },
    "modelCallsTotal": 433,
    "outputLimitStops": 3,
    "runsWithOutputLimitStop": 3,
    "refusalStops": 41,
    "runsWithRefusalStop": 2
   }
  },
  "deepagents": {
   "runs": {
    "scheduled": 12,
    "completed": 12,
    "withReport": 12,
    "retries": 0,
    "policyViolations": 0
   },
   "scores": {
    "total": {
     "jev": 0.5484894212884582,
     "astra": 0.5759732584012228,
     "two_judge_mean": 0.5622313398448405,
     "consensus_lo": 0.5134292879928983,
     "consensus_hi": 0.6071025740867197
    },
    "adapted_no_citation": {
     "astra": 0.5892082324619052,
     "jev": 0.5549780396105626,
     "two_judge_mean": 0.5720931360362339
    },
    "adapted_citation_dependent": {
     "astra": 0.5398013750954926,
     "jev": 0.5398013750954926,
     "two_judge_mean": 0.5398013750954926
    },
    "adapted_content": {
     "astra": 0.4080077030812325,
     "jev": 0.42153711484593837,
     "two_judge_mean": 0.41477240896358547
    },
    "adapted_coverage": {
     "astra": 0.6407479370291882,
     "jev": 0.6187893031330304,
     "two_judge_mean": 0.6297686200811092
    },
    "adapted_presentation": {
     "astra": 0.7736440030557677,
     "jev": 0.6545963840081487,
     "two_judge_mean": 0.7141201935319583
    },
    "adapted_citation": {
     "astra": 0.32,
     "jev": 0.12,
     "two_judge_mean": 0.22000000000000003
    }
   },
   "errorsPresentMean": {
    "astra": 0.5833333333333334,
    "jev": 0.5
   },
   "judgments": {
    "expected": 320,
    "disagree": 27,
    "cannotJudge": 1,
    "unresolvedError": 0,
    "unresolved": 28,
    "resolvedByAiPass": 8
   },
   "cost": {
    "modelTotal": 92.8282,
    "modelMean": 7.73568475,
    "modelMedian": 7.021933499999999,
    "modelMax": 14.746273,
    "retrievalTotal": 3.464,
    "retrievalMean": 0.2886666666666667,
    "generationTotal": 96.292217,
    "generationMean": 8.024351416666667,
    "generationMedian": 7.2939335,
    "generationMin": 4.349515,
    "generationMax": 15.146273,
    "meanByTokenKind": {
     "input": 0.0006226666666666667,
     "cacheWrite": 2.9197583333333337,
     "cacheRead": 0.13694706666666667,
     "output": 4.678356666666667
    }
   },
   "time": {
    "meanSeconds": 766.9616666666667,
    "medianSeconds": 708.235,
    "minSeconds": 457.06,
    "maxSeconds": 1275.31
   },
   "activity": {
    "modelCallsMean": 30.833333333333332,
    "researchCallsMean": 72.16666666666667,
    "searchCallsMean": 36.083333333333336,
    "fetchCallsMean": 36.083333333333336,
    "reportCharsMean": 28415.416666666668,
    "outputTokensMean": 116958.91666666667,
    "inputTokensMean": 976789.0,
    "tokensMean": {
     "input": 77.83333333333333,
     "cacheWrite": 291975.8333333333,
     "cacheRead": 684735.3333333334,
     "output": 116958.91666666667
    },
    "modelCallsTotal": 370,
    "outputLimitStops": 10,
    "runsWithOutputLimitStop": 6,
    "refusalStops": 1,
    "runsWithRefusalStop": 1
   }
  }
 },
 "comparisons": [
  {
   "system": "plain_tools",
   "baseline": "closed_book",
   "directionSupported": "none",
   "pHolm": 0.29217078292170784,
   "metrics": {
    "adapted_no_citation": {
     "twoJudgeMean": {
      "mean": -0.056068791074011366,
      "lo": -0.10025893367220401,
      "hi": -0.017310139039364732,
      "wins": 4,
      "losses": 8,
      "ties": 0
     },
     "jev": {
      "mean": -0.05452520317151555,
      "lo": -0.11369245021734686,
      "hi": -0.00016787636903833504,
      "wins": 3,
      "losses": 9,
      "ties": 0
     },
     "astra": {
      "mean": -0.057612378976507166,
      "lo": -0.09679171424787841,
      "hi": -0.007856340868279823,
      "wins": 1,
      "losses": 11,
      "ties": 0
     },
     "worstCase": {
      "mean": -0.17595705362024563,
      "lo": -0.2226271598065853,
      "hi": -0.13196833511919573,
      "wins": 0,
      "losses": 12,
      "ties": 0
     },
     "bestCase": {
      "mean": 0.06985811881521807,
      "lo": 0.006407930849499183,
      "hi": 0.13971408442148578,
      "wins": 9,
      "losses": 3,
      "ties": 0
     },
     "pSignFlip": 0.0206979302069793
    },
    "total": {
     "twoJudgeMean": {
      "mean": -0.040067673563145834,
      "lo": -0.09121978750709876,
      "hi": 0.009529278941237257,
      "wins": 4,
      "losses": 8,
      "ties": 0
     },
     "jev": {
      "mean": -0.03737685385712338,
      "lo": -0.10068470868957549,
      "hi": 0.019386314260089955,
      "wins": 4,
      "losses": 7,
      "ties": 1
     },
     "astra": {
      "mean": -0.042758493269168286,
      "lo": -0.08862359732998111,
      "hi": 0.015139883410298767,
      "wins": 1,
      "losses": 10,
      "ties": 1
     },
     "worstCase": {
      "mean": -0.15018181331616257,
      "lo": -0.197469347918286,
      "hi": -0.10921266781794248,
      "wins": 0,
      "losses": 12,
      "ties": 0
     },
     "bestCase": {
      "mean": 0.08339055755034754,
      "lo": 0.015389292245261418,
      "hi": 0.154687647269915,
      "wins": 9,
      "losses": 3,
      "ties": 0
     },
     "pSignFlip": 0.16768323167683233
    }
   },
   "withAiPass": {
    "worstCase": {
     "mean": -0.11896672134149609,
     "lo": -0.16907079102742004,
     "hi": -0.06611102877772677,
     "wins": 1,
     "losses": 11,
     "ties": 0
    },
    "bestCase": {
     "mean": 0.029050620930708173,
     "lo": -0.03888558486999017,
     "hi": 0.10823889896512784,
     "wins": 7,
     "losses": 5,
     "ties": 0
    },
    "directionSupported": "none"
   }
  },
  {
   "system": "gpt_researcher",
   "baseline": "plain_tools",
   "directionSupported": "none",
   "pHolm": 0.0044995500449955,
   "metrics": {
    "adapted_no_citation": {
     "twoJudgeMean": {
      "mean": -0.08461591480095398,
      "lo": -0.1421997802987418,
      "hi": -0.03626804688954286,
      "wins": 2,
      "losses": 10,
      "ties": 0
     },
     "jev": {
      "mean": -0.1012898189344144,
      "lo": -0.16310128854271566,
      "hi": -0.04413089608022279,
      "wins": 2,
      "losses": 8,
      "ties": 2
     },
     "astra": {
      "mean": -0.06794201066749354,
      "lo": -0.12917851330310479,
      "hi": -0.01877370248468966,
      "wins": 3,
      "losses": 9,
      "ties": 0
     },
     "worstCase": {
      "mean": -0.22247681985872622,
      "lo": -0.29084202954998273,
      "hi": -0.1584061788341223,
      "wins": 0,
      "losses": 12,
      "ties": 0
     },
     "bestCase": {
      "mean": 0.05026879978062777,
      "lo": -0.027781485878544517,
      "hi": 0.1315252836085091,
      "wins": 7,
      "losses": 4,
      "ties": 1
     },
     "pSignFlip": 0.0038996100389961006
    },
    "total": {
     "twoJudgeMean": {
      "mean": -0.08487949211189398,
      "lo": -0.1383301496631782,
      "hi": -0.040206144013669294,
      "wins": 1,
      "losses": 11,
      "ties": 0
     },
     "jev": {
      "mean": -0.09710874752523126,
      "lo": -0.15707875902339175,
      "hi": -0.04283911710868651,
      "wins": 0,
      "losses": 10,
      "ties": 2
     },
     "astra": {
      "mean": -0.07265023669855672,
      "lo": -0.1294726652711516,
      "hi": -0.02523698463964543,
      "wins": 2,
      "losses": 10,
      "ties": 0
     },
     "worstCase": {
      "mean": -0.22804909944453103,
      "lo": -0.2928964933890346,
      "hi": -0.16777873447737654,
      "wins": 0,
      "losses": 12,
      "ties": 0
     },
     "bestCase": {
      "mean": 0.04778297735511463,
      "lo": -0.01414616924288827,
      "hi": 0.10926048315758541,
      "wins": 7,
      "losses": 4,
      "ties": 1
     },
     "pSignFlip": 0.0014998500149985001
    }
   },
   "withAiPass": {
    "worstCase": {
     "mean": -0.17445976754011183,
     "lo": -0.24867715837424406,
     "hi": -0.09897405216563435,
     "wins": 2,
     "losses": 10,
     "ties": 0
    },
    "bestCase": {
     "mean": 0.0014955466967804948,
     "lo": -0.05095440822273142,
     "hi": 0.04744209557573367,
     "wins": 6,
     "losses": 6,
     "ties": 0
    },
    "directionSupported": "none"
   }
  },
  {
   "system": "deepagents",
   "baseline": "plain_tools",
   "directionSupported": "none",
   "pHolm": 0.29217078292170784,
   "metrics": {
    "adapted_no_citation": {
     "twoJudgeMean": {
      "mean": 0.05229581280613155,
      "lo": -0.0035374574216432628,
      "hi": 0.10596774433674337,
      "wins": 10,
      "losses": 2,
      "ties": 0
     },
     "jev": {
      "mean": 0.03463341834279536,
      "lo": -0.024833609324630836,
      "hi": 0.09353005274491273,
      "wins": 7,
      "losses": 3,
      "ties": 2
     },
     "astra": {
      "mean": 0.06995820726946775,
      "lo": 0.014324365416427582,
      "hi": 0.12389016328538144,
      "wins": 10,
      "losses": 2,
      "ties": 0
     },
     "worstCase": {
      "mean": -0.06129118082743227,
      "lo": -0.12606555532787125,
      "hi": 0.0015849314039772244,
      "wins": 2,
      "losses": 10,
      "ties": 0
     },
     "bestCase": {
      "mean": 0.16588280643969536,
      "lo": 0.08345001817321047,
      "hi": 0.26136016147135444,
      "wins": 11,
      "losses": 1,
      "ties": 0
     },
     "pSignFlip": 0.10118988101189881
    },
    "total": {
     "twoJudgeMean": {
      "mean": 0.044221029828748544,
      "lo": -0.00974432326420592,
      "hi": 0.09671586816572701,
      "wins": 9,
      "losses": 3,
      "ties": 0
     },
     "jev": {
      "mean": 0.028645558702708903,
      "lo": -0.028883647757697934,
      "hi": 0.08629346992548682,
      "wins": 7,
      "losses": 4,
      "ties": 1
     },
     "astra": {
      "mean": 0.059796500954788195,
      "lo": 0.0065832054883987155,
      "hi": 0.1108781653005677,
      "wins": 9,
      "losses": 3,
      "ties": 0
     },
     "worstCase": {
      "mean": -0.06769053657921864,
      "lo": -0.13228333901731087,
      "hi": -0.003780417944799984,
      "wins": 2,
      "losses": 10,
      "ties": 0
     },
     "bestCase": {
      "mean": 0.14827096101658999,
      "lo": 0.08051889308518918,
      "hi": 0.22108279936329991,
      "wins": 11,
      "losses": 1,
      "ties": 0
     },
     "pSignFlip": 0.14608539146085392
    }
   },
   "withAiPass": {
    "worstCase": {
     "mean": -0.008880013132853186,
     "lo": -0.07511558012846396,
     "hi": 0.054809289767025385,
     "wins": 4,
     "losses": 7,
     "ties": 1
    },
    "bestCase": {
     "mean": 0.11400708855411863,
     "lo": 0.06013922899375682,
     "hi": 0.16427154597562768,
     "wins": 10,
     "losses": 1,
     "ties": 1
    },
    "directionSupported": "none"
   }
  }
 ],
 "tasks": [
  {
   "id": "rr-6053c9",
   "domain": "Technical Documentation",
   "timeSensitive": false,
   "breadth": "Moderate",
   "nesting": "Deep",
   "exploration": "Medium",
   "criteria": 29,
   "errorCriteria": 3,
   "citationDependentCriteria": 2
  },
  {
   "id": "rr-6053f5",
   "domain": "Other",
   "timeSensitive": true,
   "breadth": "Moderate",
   "nesting": "Deep",
   "exploration": "Medium",
   "criteria": 22,
   "errorCriteria": 2,
   "citationDependentCriteria": 1
  },
  {
   "id": "rr-60542a",
   "domain": "Technical Documentation",
   "timeSensitive": true,
   "breadth": "Moderate",
   "nesting": "Intermediate",
   "exploration": "Medium",
   "criteria": 27,
   "errorCriteria": 3,
   "citationDependentCriteria": 1
  },
  {
   "id": "rr-605433",
   "domain": "Business Planning & Research",
   "timeSensitive": true,
   "breadth": "Moderate",
   "nesting": "Intermediate",
   "exploration": "High",
   "criteria": 28,
   "errorCriteria": 2,
   "citationDependentCriteria": 1
  },
  {
   "id": "rr-60543e",
   "domain": "General Consumer Research",
   "timeSensitive": true,
   "breadth": "Simple",
   "nesting": "Intermediate",
   "exploration": "Low",
   "criteria": 28,
   "errorCriteria": 2,
   "citationDependentCriteria": 7
  },
  {
   "id": "rr-605441",
   "domain": "Current Events",
   "timeSensitive": true,
   "breadth": "Simple",
   "nesting": "Shallow",
   "exploration": "Medium",
   "criteria": 28,
   "errorCriteria": 2,
   "citationDependentCriteria": 4
  },
  {
   "id": "rr-605469",
   "domain": "Hypotheticals & Philosophy",
   "timeSensitive": false,
   "breadth": "High",
   "nesting": "Intermediate",
   "exploration": "Medium",
   "criteria": 34,
   "errorCriteria": 2,
   "citationDependentCriteria": 1
  },
  {
   "id": "rr-60548a",
   "domain": "AI & ML",
   "timeSensitive": false,
   "breadth": "Moderate",
   "nesting": "Intermediate",
   "exploration": "Medium",
   "criteria": 21,
   "errorCriteria": 2,
   "citationDependentCriteria": 1
  },
  {
   "id": "rr-9af308",
   "domain": "STEM",
   "timeSensitive": false,
   "breadth": "Simple",
   "nesting": "Intermediate",
   "exploration": "Low",
   "criteria": 24,
   "errorCriteria": 5,
   "citationDependentCriteria": 1
  },
  {
   "id": "rr-9af316",
   "domain": "Current Events",
   "timeSensitive": true,
   "breadth": "Simple",
   "nesting": "Intermediate",
   "exploration": "Medium",
   "criteria": 24,
   "errorCriteria": 3,
   "citationDependentCriteria": 7
  },
  {
   "id": "rr-9af31d",
   "domain": "Historical Analysis",
   "timeSensitive": false,
   "breadth": "Moderate",
   "nesting": "Intermediate",
   "exploration": "Medium",
   "criteria": 32,
   "errorCriteria": 4,
   "citationDependentCriteria": 3
  },
  {
   "id": "rr-9af332",
   "domain": "Creative Writing",
   "timeSensitive": false,
   "breadth": "Simple",
   "nesting": "Shallow",
   "exploration": "Low",
   "criteria": 23,
   "errorCriteria": 5,
   "citationDependentCriteria": 2
  }
 ],
 "runs": [
  {
   "taskId": "rr-605441",
   "system": "deepagents",
   "block": 1,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.36792452830188677,
     "adapted_no_citation": 0.4235294117647059,
     "adapted_citation_dependent": 0.14285714285714285
    },
    "astra": {
     "total": 0.3584905660377358,
     "adapted_no_citation": 0.4117647058823529,
     "adapted_citation_dependent": 0.14285714285714285
    },
    "two_judge_mean": {
     "total": 0.3632075471698113,
     "adapted_no_citation": 0.41764705882352937,
     "adapted_citation_dependent": 0.14285714285714285
    },
    "consensus_lo": {
     "total": 0.27358490566037735,
     "adapted_no_citation": 0.3058823529411765,
     "adapted_citation_dependent": 0.14285714285714285
    },
    "consensus_hi": {
     "total": 0.4056603773584906,
     "adapted_no_citation": 0.47058823529411764,
     "adapted_citation_dependent": 0.14285714285714285
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 4.189515,
   "retrievalUsd": 0.16,
   "wallSeconds": 457.06,
   "modelCalls": 19,
   "searchCalls": 20,
   "fetchCalls": 20,
   "reportChars": 14557,
   "tokens": {
    "input": 48,
    "cacheWrite": 160848,
    "cacheRead": 274654,
    "output": 63143
   },
   "costByTokenKind": {
    "input": 0.000384,
    "cacheWrite": 1.60848,
    "cacheRead": 0.0549308,
    "output": 2.52572
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 2,
    "cannotJudge": 1,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-605441",
   "system": "gpt_researcher",
   "block": 1,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.37735849056603776,
     "adapted_no_citation": 0.35294117647058826,
     "adapted_citation_dependent": 0.47619047619047616
    },
    "astra": {
     "total": 0.29245283018867924,
     "adapted_no_citation": 0.36470588235294116,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.33490566037735847,
     "adapted_no_citation": 0.3588235294117647,
     "adapted_citation_dependent": 0.23809523809523808
    },
    "consensus_lo": {
     "total": 0.16981132075471697,
     "adapted_no_citation": 0.21176470588235294,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.4528301886792453,
     "adapted_no_citation": 0.4470588235294118,
     "adapted_citation_dependent": 0.47619047619047616
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 13.331976,
   "retrievalUsd": 0.504,
   "wallSeconds": 547.01,
   "modelCalls": 61,
   "searchCalls": 42,
   "fetchCalls": 210,
   "reportChars": 22885,
   "tokens": {
    "input": 1410317,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 51236
   },
   "costByTokenKind": {
    "input": 11.282536,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.04944
   },
   "outputLimitStops": 0,
   "refusalStops": 30,
   "judgments": {
    "expected": 28,
    "disagree": 5,
    "cannotJudge": 1,
    "unresolved": 6
   }
  },
  {
   "taskId": "rr-605441",
   "system": "plain_tools",
   "block": 1,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.39622641509433965,
     "adapted_no_citation": 0.4588235294117647,
     "adapted_citation_dependent": 0.14285714285714285
    },
    "astra": {
     "total": 0.36792452830188677,
     "adapted_no_citation": 0.4235294117647059,
     "adapted_citation_dependent": 0.14285714285714285
    },
    "two_judge_mean": {
     "total": 0.3820754716981132,
     "adapted_no_citation": 0.4411764705882353,
     "adapted_citation_dependent": 0.14285714285714285
    },
    "consensus_lo": {
     "total": 0.2830188679245283,
     "adapted_no_citation": 0.3176470588235294,
     "adapted_citation_dependent": 0.14285714285714285
    },
    "consensus_hi": {
     "total": 0.5283018867924528,
     "adapted_no_citation": 0.5058823529411764,
     "adapted_citation_dependent": 0.6190476190476191
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 1.412552,
   "retrievalUsd": 0.128,
   "wallSeconds": 124.62,
   "modelCalls": 7,
   "searchCalls": 15,
   "fetchCalls": 23,
   "reportChars": 7125,
   "tokens": {
    "input": 16,
    "cacheWrite": 99755,
    "cacheRead": 230770,
    "output": 9218
   },
   "costByTokenKind": {
    "input": 0.000128,
    "cacheWrite": 0.99755,
    "cacheRead": 0.046154,
    "output": 0.36872
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 4,
    "cannotJudge": 2,
    "unresolved": 5
   }
  },
  {
   "taskId": "rr-605441",
   "system": "closed_book",
   "block": 1,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.5,
     "adapted_no_citation": 0.5411764705882353,
     "adapted_citation_dependent": 0.3333333333333333
    },
    "astra": {
     "total": 0.4528301886792453,
     "adapted_no_citation": 0.4823529411764706,
     "adapted_citation_dependent": 0.3333333333333333
    },
    "two_judge_mean": {
     "total": 0.47641509433962265,
     "adapted_no_citation": 0.5117647058823529,
     "adapted_citation_dependent": 0.3333333333333333
    },
    "consensus_lo": {
     "total": 0.36792452830188677,
     "adapted_no_citation": 0.3764705882352941,
     "adapted_citation_dependent": 0.3333333333333333
    },
    "consensus_hi": {
     "total": 0.5377358490566038,
     "adapted_no_citation": 0.5882352941176471,
     "adapted_citation_dependent": 0.3333333333333333
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 0.20396,
   "retrievalUsd": 0.0,
   "wallSeconds": 53.7,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 6173,
   "tokens": {
    "input": 220,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 5055
   },
   "costByTokenKind": {
    "input": 0.00176,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.2022
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 3,
    "cannotJudge": 1,
    "unresolved": 4
   }
  },
  {
   "taskId": "rr-605433",
   "system": "plain_tools",
   "block": 3,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.8529411764705882,
     "adapted_no_citation": 0.8787878787878788,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.8823529411764706,
     "adapted_no_citation": 0.8787878787878788,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.8676470588235294,
     "adapted_no_citation": 0.8787878787878788,
     "adapted_citation_dependent": 0.5
    },
    "consensus_lo": {
     "total": 0.8529411764705882,
     "adapted_no_citation": 0.8787878787878788,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.8823529411764706,
     "adapted_no_citation": 0.8787878787878788,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 2.720578,
   "retrievalUsd": 0.183,
   "wallSeconds": 477.09,
   "modelCalls": 7,
   "searchCalls": 24,
   "fetchCalls": 15,
   "reportChars": 14219,
   "tokens": {
    "input": 16,
    "cacheWrite": 115829,
    "cacheRead": 325199,
    "output": 37428
   },
   "costByTokenKind": {
    "input": 0.000128,
    "cacheWrite": 1.15829,
    "cacheRead": 0.06503980000000001,
    "output": 1.49712
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 1,
    "cannotJudge": 0,
    "unresolved": 1
   }
  },
  {
   "taskId": "rr-605433",
   "system": "deepagents",
   "block": 3,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.6470588235294118,
     "adapted_no_citation": 0.6666666666666666,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.75,
     "adapted_no_citation": 0.7424242424242424,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.6985294117647058,
     "adapted_no_citation": 0.7045454545454546,
     "adapted_citation_dependent": 0.5
    },
    "consensus_lo": {
     "total": 0.6176470588235294,
     "adapted_no_citation": 0.6363636363636364,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.7794117647058824,
     "adapted_no_citation": 0.7727272727272727,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 4.382055,
   "retrievalUsd": 0.24,
   "wallSeconds": 514.68,
   "modelCalls": 22,
   "searchCalls": 30,
   "fetchCalls": 30,
   "reportChars": 18031,
   "tokens": {
    "input": 58,
    "cacheWrite": 135208,
    "cacheRead": 235354,
    "output": 74561
   },
   "costByTokenKind": {
    "input": 0.000464,
    "cacheWrite": 1.35208,
    "cacheRead": 0.0470708,
    "output": 2.98244
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 5,
    "cannotJudge": 0,
    "unresolved": 5
   }
  },
  {
   "taskId": "rr-605433",
   "system": "gpt_researcher",
   "block": 3,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.6176470588235294,
     "adapted_no_citation": 0.6363636363636364,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.7941176470588235,
     "adapted_no_citation": 0.7878787878787878,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.7058823529411764,
     "adapted_no_citation": 0.7121212121212122,
     "adapted_citation_dependent": 0.5
    },
    "consensus_lo": {
     "total": 0.6176470588235294,
     "adapted_no_citation": 0.6363636363636364,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.7941176470588235,
     "adapted_no_citation": 0.7878787878787878,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 6.46468,
   "retrievalUsd": 0.552,
   "wallSeconds": 461.56,
   "modelCalls": 33,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 25074,
   "tokens": {
    "input": 498480,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 61921
   },
   "costByTokenKind": {
    "input": 3.98784,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.47684
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 5,
    "cannotJudge": 0,
    "unresolved": 5
   }
  },
  {
   "taskId": "rr-605433",
   "system": "closed_book",
   "block": 3,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.7794117647058824,
     "adapted_no_citation": 0.803030303030303,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.9117647058823529,
     "adapted_no_citation": 0.9393939393939394,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.8455882352941176,
     "adapted_no_citation": 0.8712121212121212,
     "adapted_citation_dependent": 0.0
    },
    "consensus_lo": {
     "total": 0.75,
     "adapted_no_citation": 0.7727272727272727,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.9411764705882353,
     "adapted_no_citation": 0.9696969696969697,
     "adapted_citation_dependent": 0.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 0.370984,
   "retrievalUsd": 0.0,
   "wallSeconds": 107.38,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 10120,
   "tokens": {
    "input": 578,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 9159
   },
   "costByTokenKind": {
    "input": 0.004624,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.36636
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 5,
    "cannotJudge": 0,
    "unresolved": 5
   }
  },
  {
   "taskId": "rr-60548a",
   "system": "closed_book",
   "block": 4,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.6875,
     "adapted_no_citation": 0.6721311475409836,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.703125,
     "adapted_no_citation": 0.7377049180327869,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.6953125,
     "adapted_no_citation": 0.7049180327868853,
     "adapted_citation_dependent": 0.5
    },
    "consensus_lo": {
     "total": 0.640625,
     "adapted_no_citation": 0.6721311475409836,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.75,
     "adapted_no_citation": 0.7377049180327869,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 0.318648,
   "retrievalUsd": 0.0,
   "wallSeconds": 74.05,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 15448,
   "tokens": {
    "input": 261,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 7914
   },
   "costByTokenKind": {
    "input": 0.002088,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.31656
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 21,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-60548a",
   "system": "deepagents",
   "block": 4,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.75,
     "adapted_no_citation": 0.7377049180327869,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.703125,
     "adapted_no_citation": 0.7377049180327869,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.7265625,
     "adapted_no_citation": 0.7377049180327869,
     "adapted_citation_dependent": 0.5
    },
    "consensus_lo": {
     "total": 0.703125,
     "adapted_no_citation": 0.7377049180327869,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.75,
     "adapted_no_citation": 0.7377049180327869,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 7.053798,
   "retrievalUsd": 0.24,
   "wallSeconds": 726.1,
   "modelCalls": 29,
   "searchCalls": 30,
   "fetchCalls": 30,
   "reportChars": 22787,
   "tokens": {
    "input": 72,
    "cacheWrite": 289790,
    "cacheRead": 758009,
    "output": 100093
   },
   "costByTokenKind": {
    "input": 0.000576,
    "cacheWrite": 2.8979,
    "cacheRead": 0.1516018,
    "output": 4.00372
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 21,
    "disagree": 1,
    "cannotJudge": 0,
    "unresolved": 1
   }
  },
  {
   "taskId": "rr-60548a",
   "system": "plain_tools",
   "block": 4,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.765625,
     "adapted_no_citation": 0.7540983606557377,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.65625,
     "adapted_no_citation": 0.6885245901639344,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.7109375,
     "adapted_no_citation": 0.721311475409836,
     "adapted_citation_dependent": 0.5
    },
    "consensus_lo": {
     "total": 0.65625,
     "adapted_no_citation": 0.6885245901639344,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.765625,
     "adapted_no_citation": 0.7540983606557377,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 1.749728,
   "retrievalUsd": 0.068,
   "wallSeconds": 112.43,
   "modelCalls": 5,
   "searchCalls": 8,
   "fetchCalls": 12,
   "reportChars": 11857,
   "tokens": {
    "input": 12,
    "cacheWrite": 136603,
    "cacheRead": 156612,
    "output": 8807
   },
   "costByTokenKind": {
    "input": 9.6e-05,
    "cacheWrite": 1.36603,
    "cacheRead": 0.0313224,
    "output": 0.35228
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 21,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-60548a",
   "system": "gpt_researcher",
   "block": 4,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.484375,
     "adapted_no_citation": 0.45901639344262296,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.34375,
     "adapted_no_citation": 0.36065573770491804,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.4140625,
     "adapted_no_citation": 0.4098360655737705,
     "adapted_citation_dependent": 0.5
    },
    "consensus_lo": {
     "total": 0.34375,
     "adapted_no_citation": 0.36065573770491804,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.484375,
     "adapted_no_citation": 0.45901639344262296,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 2
   },
   "modelUsd": 8.339456,
   "retrievalUsd": 0.552,
   "wallSeconds": 431.64,
   "modelCalls": 33,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 25730,
   "tokens": {
    "input": 745452,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 59396
   },
   "costByTokenKind": {
    "input": 5.963616,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.37584
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 21,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-9af308",
   "system": "gpt_researcher",
   "block": 5,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.7611940298507462,
     "adapted_no_citation": 0.7611940298507462,
     "adapted_citation_dependent": null
    },
    "astra": {
     "total": 0.6567164179104478,
     "adapted_no_citation": 0.6567164179104478,
     "adapted_citation_dependent": null
    },
    "two_judge_mean": {
     "total": 0.708955223880597,
     "adapted_no_citation": 0.708955223880597,
     "adapted_citation_dependent": null
    },
    "consensus_lo": {
     "total": 0.6567164179104478,
     "adapted_no_citation": 0.6567164179104478,
     "adapted_citation_dependent": null
    },
    "consensus_hi": {
     "total": 0.7611940298507462,
     "adapted_no_citation": 0.7611940298507462,
     "adapted_citation_dependent": null
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 1
   },
   "modelUsd": 6.851584,
   "retrievalUsd": 0.552,
   "wallSeconds": 428.34,
   "modelCalls": 33,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 23861,
   "tokens": {
    "input": 566343,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 58021
   },
   "costByTokenKind": {
    "input": 4.530744,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.32084
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 24,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-9af308",
   "system": "plain_tools",
   "block": 5,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.7611940298507462,
     "adapted_no_citation": 0.7611940298507462,
     "adapted_citation_dependent": null
    },
    "astra": {
     "total": 0.582089552238806,
     "adapted_no_citation": 0.582089552238806,
     "adapted_citation_dependent": null
    },
    "two_judge_mean": {
     "total": 0.6716417910447761,
     "adapted_no_citation": 0.6716417910447761,
     "adapted_citation_dependent": null
    },
    "consensus_lo": {
     "total": 0.582089552238806,
     "adapted_no_citation": 0.582089552238806,
     "adapted_citation_dependent": null
    },
    "consensus_hi": {
     "total": 0.7611940298507462,
     "adapted_no_citation": 0.7611940298507462,
     "adapted_citation_dependent": null
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 1
   },
   "modelUsd": 1.837239,
   "retrievalUsd": 0.136,
   "wallSeconds": 266.05,
   "modelCalls": 7,
   "searchCalls": 17,
   "fetchCalls": 17,
   "reportChars": 11371,
   "tokens": {
    "input": 16,
    "cacheWrite": 93690,
    "cacheRead": 209854,
    "output": 21456
   },
   "costByTokenKind": {
    "input": 0.000128,
    "cacheWrite": 0.9369,
    "cacheRead": 0.0419708,
    "output": 0.85824
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 24,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-9af308",
   "system": "deepagents",
   "block": 5,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.8059701492537313,
     "adapted_no_citation": 0.8059701492537313,
     "adapted_citation_dependent": null
    },
    "astra": {
     "total": 0.746268656716418,
     "adapted_no_citation": 0.746268656716418,
     "adapted_citation_dependent": null
    },
    "two_judge_mean": {
     "total": 0.7761194029850746,
     "adapted_no_citation": 0.7761194029850746,
     "adapted_citation_dependent": null
    },
    "consensus_lo": {
     "total": 0.746268656716418,
     "adapted_no_citation": 0.746268656716418,
     "adapted_citation_dependent": null
    },
    "consensus_hi": {
     "total": 0.8059701492537313,
     "adapted_no_citation": 0.8059701492537313,
     "adapted_citation_dependent": null
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 5.277714,
   "retrievalUsd": 0.16,
   "wallSeconds": 690.37,
   "modelCalls": 20,
   "searchCalls": 20,
   "fetchCalls": 20,
   "reportChars": 26122,
   "tokens": {
    "input": 50,
    "cacheWrite": 164872,
    "cacheRead": 360569,
    "output": 88912
   },
   "costByTokenKind": {
    "input": 0.0004,
    "cacheWrite": 1.64872,
    "cacheRead": 0.0721138,
    "output": 3.55648
   },
   "outputLimitStops": 1,
   "refusalStops": 0,
   "judgments": {
    "expected": 24,
    "disagree": 1,
    "cannotJudge": 0,
    "unresolved": 1
   }
  },
  {
   "taskId": "rr-9af308",
   "system": "closed_book",
   "block": 5,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.8059701492537313,
     "adapted_no_citation": 0.8059701492537313,
     "adapted_citation_dependent": null
    },
    "astra": {
     "total": 0.7014925373134329,
     "adapted_no_citation": 0.7014925373134329,
     "adapted_citation_dependent": null
    },
    "two_judge_mean": {
     "total": 0.7537313432835822,
     "adapted_no_citation": 0.7537313432835822,
     "adapted_citation_dependent": null
    },
    "consensus_lo": {
     "total": 0.7014925373134329,
     "adapted_no_citation": 0.7014925373134329,
     "adapted_citation_dependent": null
    },
    "consensus_hi": {
     "total": 0.8059701492537313,
     "adapted_no_citation": 0.8059701492537313,
     "adapted_citation_dependent": null
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 1
   },
   "modelUsd": 0.449016,
   "retrievalUsd": 0.0,
   "wallSeconds": 110.82,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 11583,
   "tokens": {
    "input": 237,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 11178
   },
   "costByTokenKind": {
    "input": 0.001896,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.44712
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 24,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-9af316",
   "system": "deepagents",
   "block": 7,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.5405405405405406,
     "adapted_no_citation": 0.5925925925925926,
     "adapted_citation_dependent": 0.4
    },
    "astra": {
     "total": 0.6621621621621622,
     "adapted_no_citation": 0.7592592592592593,
     "adapted_citation_dependent": 0.4
    },
    "two_judge_mean": {
     "total": 0.6013513513513513,
     "adapted_no_citation": 0.6759259259259259,
     "adapted_citation_dependent": 0.4
    },
    "consensus_lo": {
     "total": 0.4594594594594595,
     "adapted_no_citation": 0.48148148148148145,
     "adapted_citation_dependent": 0.4
    },
    "consensus_hi": {
     "total": 0.7432432432432432,
     "adapted_no_citation": 0.8703703703703703,
     "adapted_citation_dependent": 0.4
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 2
   },
   "modelUsd": 6.547504,
   "retrievalUsd": 0.32,
   "wallSeconds": 682.96,
   "modelCalls": 33,
   "searchCalls": 40,
   "fetchCalls": 40,
   "reportChars": 33340,
   "tokens": {
    "input": 84,
    "cacheWrite": 212510,
    "cacheRead": 437262,
    "output": 108357
   },
   "costByTokenKind": {
    "input": 0.000672,
    "cacheWrite": 2.1251,
    "cacheRead": 0.08745240000000001,
    "output": 4.33428
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 24,
    "disagree": 6,
    "cannotJudge": 0,
    "unresolved": 6
   }
  },
  {
   "taskId": "rr-9af316",
   "system": "gpt_researcher",
   "block": 7,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.43243243243243246,
     "adapted_no_citation": 0.5,
     "adapted_citation_dependent": 0.25
    },
    "astra": {
     "total": 0.40540540540540543,
     "adapted_no_citation": 0.46296296296296297,
     "adapted_citation_dependent": 0.25
    },
    "two_judge_mean": {
     "total": 0.41891891891891897,
     "adapted_no_citation": 0.4814814814814815,
     "adapted_citation_dependent": 0.25
    },
    "consensus_lo": {
     "total": 0.2972972972972973,
     "adapted_no_citation": 0.3148148148148148,
     "adapted_citation_dependent": 0.25
    },
    "consensus_hi": {
     "total": 0.5405405405405406,
     "adapted_no_citation": 0.6481481481481481,
     "adapted_citation_dependent": 0.25
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 2
   },
   "modelUsd": 5.792312,
   "retrievalUsd": 0.552,
   "wallSeconds": 435.05,
   "modelCalls": 33,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 33195,
   "tokens": {
    "input": 416279,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 61552
   },
   "costByTokenKind": {
    "input": 3.330232,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.46208
   },
   "outputLimitStops": 1,
   "refusalStops": 0,
   "judgments": {
    "expected": 24,
    "disagree": 5,
    "cannotJudge": 0,
    "unresolved": 5
   }
  },
  {
   "taskId": "rr-9af316",
   "system": "plain_tools",
   "block": 7,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.44594594594594594,
     "adapted_no_citation": 0.48148148148148145,
     "adapted_citation_dependent": 0.35
    },
    "astra": {
     "total": 0.4864864864864865,
     "adapted_no_citation": 0.5370370370370371,
     "adapted_citation_dependent": 0.35
    },
    "two_judge_mean": {
     "total": 0.46621621621621623,
     "adapted_no_citation": 0.5092592592592593,
     "adapted_citation_dependent": 0.35
    },
    "consensus_lo": {
     "total": 0.3108108108108108,
     "adapted_no_citation": 0.2962962962962963,
     "adapted_citation_dependent": 0.35
    },
    "consensus_hi": {
     "total": 0.6216216216216216,
     "adapted_no_citation": 0.7222222222222222,
     "adapted_citation_dependent": 0.35
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 2
   },
   "modelUsd": 2.577162,
   "retrievalUsd": 0.27,
   "wallSeconds": 268.34,
   "modelCalls": 15,
   "searchCalls": 34,
   "fetchCalls": 32,
   "reportChars": 15692,
   "tokens": {
    "input": 32,
    "cacheWrite": 158528,
    "cacheRead": 1181132,
    "output": 18885
   },
   "costByTokenKind": {
    "input": 0.000256,
    "cacheWrite": 1.58528,
    "cacheRead": 0.23622640000000003,
    "output": 0.7554
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 24,
    "disagree": 6,
    "cannotJudge": 0,
    "unresolved": 6
   }
  },
  {
   "taskId": "rr-9af316",
   "system": "closed_book",
   "block": 7,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.40540540540540543,
     "adapted_no_citation": 0.5555555555555556,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.2702702702702703,
     "adapted_no_citation": 0.37037037037037035,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.33783783783783783,
     "adapted_no_citation": 0.46296296296296297,
     "adapted_citation_dependent": 0.0
    },
    "consensus_lo": {
     "total": 0.2702702702702703,
     "adapted_no_citation": 0.37037037037037035,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.40540540540540543,
     "adapted_no_citation": 0.5555555555555556,
     "adapted_citation_dependent": 0.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 2
   },
   "modelUsd": 0.303616,
   "retrievalUsd": 0.0,
   "wallSeconds": 81.37,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 8921,
   "tokens": {
    "input": 277,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 7535
   },
   "costByTokenKind": {
    "input": 0.002216,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.3014
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 24,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-9af31d",
   "system": "deepagents",
   "block": 8,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.5921052631578947,
     "adapted_no_citation": 0.5507246376811594,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.6842105263157895,
     "adapted_no_citation": 0.6521739130434783,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.638157894736842,
     "adapted_no_citation": 0.6014492753623188,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.5921052631578947,
     "adapted_no_citation": 0.5507246376811594,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.6842105263157895,
     "adapted_no_citation": 0.6521739130434783,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 14.746273,
   "retrievalUsd": 0.4,
   "wallSeconds": 1275.31,
   "modelCalls": 40,
   "searchCalls": 50,
   "fetchCalls": 50,
   "reportChars": 27948,
   "tokens": {
    "input": 100,
    "cacheWrite": 547055,
    "cacheRead": 1093816,
    "output": 226404
   },
   "costByTokenKind": {
    "input": 0.0008,
    "cacheWrite": 5.47055,
    "cacheRead": 0.21876320000000002,
    "output": 9.05616
   },
   "outputLimitStops": 5,
   "refusalStops": 1,
   "judgments": {
    "expected": 32,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-9af31d",
   "system": "gpt_researcher",
   "block": 8,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.5263157894736842,
     "adapted_no_citation": 0.4782608695652174,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.5394736842105263,
     "adapted_no_citation": 0.5507246376811594,
     "adapted_citation_dependent": 0.42857142857142855
    },
    "two_judge_mean": {
     "total": 0.5328947368421053,
     "adapted_no_citation": 0.5144927536231885,
     "adapted_citation_dependent": 0.7142857142857143
    },
    "consensus_lo": {
     "total": 0.47368421052631576,
     "adapted_no_citation": 0.4782608695652174,
     "adapted_citation_dependent": 0.42857142857142855
    },
    "consensus_hi": {
     "total": 0.5921052631578947,
     "adapted_no_citation": 0.5507246376811594,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 1
   },
   "modelUsd": 7.294096,
   "retrievalUsd": 0.552,
   "wallSeconds": 414.99,
   "modelCalls": 33,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 23997,
   "tokens": {
    "input": 634547,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 55443
   },
   "costByTokenKind": {
    "input": 5.076376,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.21772
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 32,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-9af31d",
   "system": "closed_book",
   "block": 8,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.5657894736842105,
     "adapted_no_citation": 0.5652173913043478,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "astra": {
     "total": 0.631578947368421,
     "adapted_no_citation": 0.6376811594202898,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "two_judge_mean": {
     "total": 0.5986842105263157,
     "adapted_no_citation": 0.6014492753623188,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "consensus_lo": {
     "total": 0.47368421052631576,
     "adapted_no_citation": 0.463768115942029,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "consensus_hi": {
     "total": 0.6578947368421053,
     "adapted_no_citation": 0.6666666666666666,
     "adapted_citation_dependent": 0.5714285714285714
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 0.539584,
   "retrievalUsd": 0.0,
   "wallSeconds": 165.88,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 14771,
   "tokens": {
    "input": 343,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 13421
   },
   "costByTokenKind": {
    "input": 0.002744,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.53684
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 32,
    "disagree": 5,
    "cannotJudge": 1,
    "unresolved": 5
   }
  },
  {
   "taskId": "rr-9af31d",
   "system": "plain_tools",
   "block": 8,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.5657894736842105,
     "adapted_no_citation": 0.5217391304347826,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.618421052631579,
     "adapted_no_citation": 0.5797101449275363,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.5921052631578947,
     "adapted_no_citation": 0.5507246376811594,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.5657894736842105,
     "adapted_no_citation": 0.5217391304347826,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.618421052631579,
     "adapted_no_citation": 0.5797101449275363,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 3.635951,
   "retrievalUsd": 0.131,
   "wallSeconds": 290.42,
   "modelCalls": 10,
   "searchCalls": 15,
   "fetchCalls": 27,
   "reportChars": 19071,
   "tokens": {
    "input": 22,
    "cacheWrite": 248064,
    "cacheRead": 988275,
    "output": 23937
   },
   "costByTokenKind": {
    "input": 0.000176,
    "cacheWrite": 2.48064,
    "cacheRead": 0.197655,
    "output": 0.95748
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 32,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-605469",
   "system": "gpt_researcher",
   "block": 9,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.34146341463414637,
     "adapted_no_citation": 0.35443037974683544,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.2926829268292683,
     "adapted_no_citation": 0.3037974683544304,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.3170731707317073,
     "adapted_no_citation": 0.3291139240506329,
     "adapted_citation_dependent": 0.0
    },
    "consensus_lo": {
     "total": 0.23170731707317074,
     "adapted_no_citation": 0.24050632911392406,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.4024390243902439,
     "adapted_no_citation": 0.4177215189873418,
     "adapted_citation_dependent": 0.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 2
   },
   "modelUsd": 6.983352,
   "retrievalUsd": 0.552,
   "wallSeconds": 423.48,
   "modelCalls": 33,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 27545,
   "tokens": {
    "input": 589094,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 56765
   },
   "costByTokenKind": {
    "input": 4.712752,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.2706
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 34,
    "disagree": 5,
    "cannotJudge": 0,
    "unresolved": 5
   }
  },
  {
   "taskId": "rr-605469",
   "system": "closed_book",
   "block": 9,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.36585365853658536,
     "adapted_no_citation": 0.379746835443038,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.43902439024390244,
     "adapted_no_citation": 0.45569620253164556,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.4024390243902439,
     "adapted_no_citation": 0.4177215189873418,
     "adapted_citation_dependent": 0.0
    },
    "consensus_lo": {
     "total": 0.36585365853658536,
     "adapted_no_citation": 0.379746835443038,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.43902439024390244,
     "adapted_no_citation": 0.45569620253164556,
     "adapted_citation_dependent": 0.0
    }
   },
   "errorsPresent": {
    "jev": 2,
    "astra": 2
   },
   "modelUsd": 0.252128,
   "retrievalUsd": 0.0,
   "wallSeconds": 73.75,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 11585,
   "tokens": {
    "input": 361,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 6231
   },
   "costByTokenKind": {
    "input": 0.002888,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.24924
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 34,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-605469",
   "system": "plain_tools",
   "block": 9,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.34146341463414637,
     "adapted_no_citation": 0.35443037974683544,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.3902439024390244,
     "adapted_no_citation": 0.4050632911392405,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.36585365853658536,
     "adapted_no_citation": 0.379746835443038,
     "adapted_citation_dependent": 0.0
    },
    "consensus_lo": {
     "total": 0.3048780487804878,
     "adapted_no_citation": 0.31645569620253167,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.4268292682926829,
     "adapted_no_citation": 0.4430379746835443,
     "adapted_citation_dependent": 0.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 2
   },
   "modelUsd": 3.011892,
   "retrievalUsd": 0.267,
   "wallSeconds": 309.34,
   "modelCalls": 12,
   "searchCalls": 34,
   "fetchCalls": 29,
   "reportChars": 17943,
   "tokens": {
    "input": 26,
    "cacheWrite": 199951,
    "cacheRead": 905669,
    "output": 20776
   },
   "costByTokenKind": {
    "input": 0.000208,
    "cacheWrite": 1.99951,
    "cacheRead": 0.1811338,
    "output": 0.83104
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 34,
    "disagree": 4,
    "cannotJudge": 0,
    "unresolved": 4
   }
  },
  {
   "taskId": "rr-605469",
   "system": "deepagents",
   "block": 9,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.4024390243902439,
     "adapted_no_citation": 0.4177215189873418,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.4146341463414634,
     "adapted_no_citation": 0.43037974683544306,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.4085365853658537,
     "adapted_no_citation": 0.42405063291139244,
     "adapted_citation_dependent": 0.0
    },
    "consensus_lo": {
     "total": 0.36585365853658536,
     "adapted_no_citation": 0.379746835443038,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.45121951219512196,
     "adapted_no_citation": 0.46835443037974683,
     "adapted_citation_dependent": 0.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 2
   },
   "modelUsd": 5.698553,
   "retrievalUsd": 0.24,
   "wallSeconds": 479.37,
   "modelCalls": 31,
   "searchCalls": 30,
   "fetchCalls": 30,
   "reportChars": 35179,
   "tokens": {
    "input": 76,
    "cacheWrite": 261305,
    "cacheRead": 847073,
    "output": 72887
   },
   "costByTokenKind": {
    "input": 0.000608,
    "cacheWrite": 2.61305,
    "cacheRead": 0.1694146,
    "output": 2.91548
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 34,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-9af332",
   "system": "deepagents",
   "block": 10,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.6666666666666666,
     "adapted_no_citation": 0.6785714285714286,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "astra": {
     "total": 0.6666666666666666,
     "adapted_no_citation": 0.6785714285714286,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "two_judge_mean": {
     "total": 0.6666666666666666,
     "adapted_no_citation": 0.6785714285714286,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "consensus_lo": {
     "total": 0.6666666666666666,
     "adapted_no_citation": 0.6785714285714286,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "consensus_hi": {
     "total": 0.6666666666666666,
     "adapted_no_citation": 0.6785714285714286,
     "adapted_citation_dependent": 0.5714285714285714
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 10.741911,
   "retrievalUsd": 0.36,
   "wallSeconds": 1060.71,
   "modelCalls": 36,
   "searchCalls": 45,
   "fetchCalls": 45,
   "reportChars": 23734,
   "tokens": {
    "input": 92,
    "cacheWrite": 413309,
    "cacheRead": 927824,
    "output": 160563
   },
   "costByTokenKind": {
    "input": 0.000736,
    "cacheWrite": 4.13309,
    "cacheRead": 0.18556480000000003,
    "output": 6.42252
   },
   "outputLimitStops": 1,
   "refusalStops": 0,
   "judgments": {
    "expected": 23,
    "disagree": 0,
    "cannotJudge": 0,
    "unresolved": 0
   }
  },
  {
   "taskId": "rr-9af332",
   "system": "closed_book",
   "block": 10,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.6031746031746031,
     "adapted_no_citation": 0.6071428571428571,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "astra": {
     "total": 0.6984126984126984,
     "adapted_no_citation": 0.7142857142857143,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "two_judge_mean": {
     "total": 0.6507936507936507,
     "adapted_no_citation": 0.6607142857142857,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "consensus_lo": {
     "total": 0.6031746031746031,
     "adapted_no_citation": 0.6071428571428571,
     "adapted_citation_dependent": 0.5714285714285714
    },
    "consensus_hi": {
     "total": 0.6984126984126984,
     "adapted_no_citation": 0.7142857142857143,
     "adapted_citation_dependent": 0.5714285714285714
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 0.501552,
   "retrievalUsd": 0.0,
   "wallSeconds": 154.15,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 10088,
   "tokens": {
    "input": 249,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 12489
   },
   "costByTokenKind": {
    "input": 0.001992,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.49956
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 23,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-9af332",
   "system": "plain_tools",
   "block": 10,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.7142857142857143,
     "adapted_no_citation": 0.6785714285714286,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.6984126984126984,
     "adapted_no_citation": 0.6607142857142857,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.7063492063492063,
     "adapted_no_citation": 0.6696428571428572,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.6349206349206349,
     "adapted_no_citation": 0.5892857142857143,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.7777777777777778,
     "adapted_no_citation": 0.75,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 3.265356,
   "retrievalUsd": 0.257,
   "wallSeconds": 510.17,
   "modelCalls": 10,
   "searchCalls": 34,
   "fetchCalls": 21,
   "reportChars": 16348,
   "tokens": {
    "input": 22,
    "cacheWrite": 162442,
    "cacheRead": 640399,
    "output": 37817
   },
   "costByTokenKind": {
    "input": 0.000176,
    "cacheWrite": 1.62442,
    "cacheRead": 0.1280798,
    "output": 1.51268
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 23,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-9af332",
   "system": "gpt_researcher",
   "block": 10,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.6031746031746031,
     "adapted_no_citation": 0.5535714285714286,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.7142857142857143,
     "adapted_no_citation": 0.6785714285714286,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.6587301587301587,
     "adapted_no_citation": 0.6160714285714286,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.5714285714285714,
     "adapted_no_citation": 0.5178571428571429,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.7142857142857143,
     "adapted_no_citation": 0.6785714285714286,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 5.240984,
   "retrievalUsd": 0.552,
   "wallSeconds": 419.69,
   "modelCalls": 33,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 29393,
   "tokens": {
    "input": 397848,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 51455
   },
   "costByTokenKind": {
    "input": 3.182784,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.0582
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 23,
    "disagree": 3,
    "cannotJudge": 2,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-6053f5",
   "system": "gpt_researcher",
   "block": 11,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.2909090909090909,
     "adapted_no_citation": 0.3076923076923077,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.41818181818181815,
     "adapted_no_citation": 0.38461538461538464,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.3545454545454545,
     "adapted_no_citation": 0.34615384615384615,
     "adapted_citation_dependent": 0.5
    },
    "consensus_lo": {
     "total": 0.2909090909090909,
     "adapted_no_citation": 0.3076923076923077,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.41818181818181815,
     "adapted_no_citation": 0.38461538461538464,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 7.094576,
   "retrievalUsd": 0.552,
   "wallSeconds": 486.74,
   "modelCalls": 42,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 25259,
   "tokens": {
    "input": 619647,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 53435
   },
   "costByTokenKind": {
    "input": 4.957176,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.1374
   },
   "outputLimitStops": 0,
   "refusalStops": 11,
   "judgments": {
    "expected": 22,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-6053f5",
   "system": "plain_tools",
   "block": 11,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.4727272727272727,
     "adapted_no_citation": 0.4423076923076923,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.45454545454545453,
     "adapted_no_citation": 0.4230769230769231,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.4636363636363636,
     "adapted_no_citation": 0.4326923076923077,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.43636363636363634,
     "adapted_no_citation": 0.40384615384615385,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.4909090909090909,
     "adapted_no_citation": 0.46153846153846156,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 2.629727,
   "retrievalUsd": 0.167,
   "wallSeconds": 310.18,
   "modelCalls": 9,
   "searchCalls": 21,
   "fetchCalls": 20,
   "reportChars": 16222,
   "tokens": {
    "input": 20,
    "cacheWrite": 158622,
    "cacheRead": 632335,
    "output": 22922
   },
   "costByTokenKind": {
    "input": 0.00016,
    "cacheWrite": 1.58622,
    "cacheRead": 0.126467,
    "output": 0.91688
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 22,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-6053f5",
   "system": "deepagents",
   "block": 11,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.5454545454545454,
     "adapted_no_citation": 0.5192307692307693,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.5272727272727272,
     "adapted_no_citation": 0.5,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.5363636363636364,
     "adapted_no_citation": 0.5096153846153846,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.4727272727272727,
     "adapted_no_citation": 0.4423076923076923,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.6,
     "adapted_no_citation": 0.5769230769230769,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 9.237776,
   "retrievalUsd": 0.36,
   "wallSeconds": 850.61,
   "modelCalls": 35,
   "searchCalls": 45,
   "fetchCalls": 45,
   "reportChars": 33749,
   "tokens": {
    "input": 90,
    "cacheWrite": 368862,
    "cacheRead": 707378,
    "output": 135174
   },
   "costByTokenKind": {
    "input": 0.00072,
    "cacheWrite": 3.68862,
    "cacheRead": 0.1414756,
    "output": 5.40696
   },
   "outputLimitStops": 1,
   "refusalStops": 0,
   "judgments": {
    "expected": 22,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-6053f5",
   "system": "closed_book",
   "block": 11,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.5818181818181818,
     "adapted_no_citation": 0.5576923076923077,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.5454545454545454,
     "adapted_no_citation": 0.5192307692307693,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.5636363636363636,
     "adapted_no_citation": 0.5384615384615385,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.5454545454545454,
     "adapted_no_citation": 0.5192307692307693,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.5818181818181818,
     "adapted_no_citation": 0.5576923076923077,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 0.800752,
   "retrievalUsd": 0.0,
   "wallSeconds": 207.57,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 20416,
   "tokens": {
    "input": 399,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 19939
   },
   "costByTokenKind": {
    "input": 0.003192,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.79756
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 22,
    "disagree": 1,
    "cannotJudge": 0,
    "unresolved": 1
   }
  },
  {
   "taskId": "rr-6053c9",
   "system": "deepagents",
   "block": 12,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.5675675675675675,
     "adapted_no_citation": 0.5362318840579711,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.7027027027027027,
     "adapted_no_citation": 0.6811594202898551,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.6351351351351351,
     "adapted_no_citation": 0.6086956521739131,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.5675675675675675,
     "adapted_no_citation": 0.5362318840579711,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.7027027027027027,
     "adapted_no_citation": 0.6811594202898551,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 0
   },
   "modelUsd": 8.839283,
   "retrievalUsd": 0.32,
   "wallSeconds": 910.3,
   "modelCalls": 35,
   "searchCalls": 40,
   "fetchCalls": 40,
   "reportChars": 31494,
   "tokens": {
    "input": 88,
    "cacheWrite": 342750,
    "cacheRead": 953797,
    "output": 130508
   },
   "costByTokenKind": {
    "input": 0.000704,
    "cacheWrite": 3.4275,
    "cacheRead": 0.19075940000000002,
    "output": 5.22032
   },
   "outputLimitStops": 1,
   "refusalStops": 0,
   "judgments": {
    "expected": 29,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-6053c9",
   "system": "gpt_researcher",
   "block": 12,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.24324324324324326,
     "adapted_no_citation": 0.18840579710144928,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.3918918918918919,
     "adapted_no_citation": 0.34782608695652173,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.31756756756756754,
     "adapted_no_citation": 0.2681159420289855,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.1891891891891892,
     "adapted_no_citation": 0.13043478260869565,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.44594594594594594,
     "adapted_no_citation": 0.4057971014492754,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 0
   },
   "modelUsd": 6.250008,
   "retrievalUsd": 0.552,
   "wallSeconds": 423.8,
   "modelCalls": 33,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 27708,
   "tokens": {
    "input": 491716,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 57907
   },
   "costByTokenKind": {
    "input": 3.933728,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.31628
   },
   "outputLimitStops": 1,
   "refusalStops": 0,
   "judgments": {
    "expected": 29,
    "disagree": 7,
    "cannotJudge": 0,
    "unresolved": 7
   }
  },
  {
   "taskId": "rr-6053c9",
   "system": "plain_tools",
   "block": 12,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.47297297297297297,
     "adapted_no_citation": 0.43478260869565216,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.5405405405405406,
     "adapted_no_citation": 0.5072463768115942,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.5067567567567568,
     "adapted_no_citation": 0.47101449275362317,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.47297297297297297,
     "adapted_no_citation": 0.43478260869565216,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.5405405405405406,
     "adapted_no_citation": 0.5072463768115942,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 2.728516,
   "retrievalUsd": 0.192,
   "wallSeconds": 271.47,
   "modelCalls": 9,
   "searchCalls": 24,
   "fetchCalls": 24,
   "reportChars": 16383,
   "tokens": {
    "input": 20,
    "cacheWrite": 185789,
    "cacheRead": 650932,
    "output": 18507
   },
   "costByTokenKind": {
    "input": 0.00016,
    "cacheWrite": 1.85789,
    "cacheRead": 0.1301864,
    "output": 0.74028
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 29,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-6053c9",
   "system": "closed_book",
   "block": 12,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.6081081081081081,
     "adapted_no_citation": 0.5797101449275363,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.6351351351351351,
     "adapted_no_citation": 0.6086956521739131,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.6216216216216216,
     "adapted_no_citation": 0.5942028985507246,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.581081081081081,
     "adapted_no_citation": 0.5507246376811594,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.6621621621621622,
     "adapted_no_citation": 0.6376811594202898,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 0.525992,
   "retrievalUsd": 0.0,
   "wallSeconds": 135.42,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 17961,
   "tokens": {
    "input": 264,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 13097
   },
   "costByTokenKind": {
    "input": 0.002112,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.52388
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 29,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-60543e",
   "system": "deepagents",
   "block": 13,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.010752688172043012,
     "adapted_no_citation": 0.05263157894736842,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "astra": {
     "total": 0.010752688172043012,
     "adapted_no_citation": 0.05263157894736842,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "two_judge_mean": {
     "total": 0.010752688172043012,
     "adapted_no_citation": 0.05263157894736842,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "consensus_lo": {
     "total": 0.010752688172043012,
     "adapted_no_citation": 0.05263157894736842,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "consensus_hi": {
     "total": 0.010752688172043012,
     "adapted_no_citation": 0.05263157894736842,
     "adapted_citation_dependent": -0.17647058823529413
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 9.123766,
   "retrievalUsd": 0.36,
   "wallSeconds": 872.37,
   "modelCalls": 40,
   "searchCalls": 45,
   "fetchCalls": 45,
   "reportChars": 44065,
   "tokens": {
    "input": 100,
    "cacheWrite": 331566,
    "cacheRead": 969932,
    "output": 140333
   },
   "costByTokenKind": {
    "input": 0.0008,
    "cacheWrite": 3.31566,
    "cacheRead": 0.19398640000000003,
    "output": 5.61332
   },
   "outputLimitStops": 1,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 0,
    "cannotJudge": 0,
    "unresolved": 0
   }
  },
  {
   "taskId": "rr-60543e",
   "system": "closed_book",
   "block": 13,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.06451612903225806,
     "adapted_no_citation": 0.11842105263157894,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "astra": {
     "total": 0.021505376344086023,
     "adapted_no_citation": 0.06578947368421052,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "two_judge_mean": {
     "total": 0.043010752688172046,
     "adapted_no_citation": 0.09210526315789473,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "consensus_lo": {
     "total": 0.021505376344086023,
     "adapted_no_citation": 0.06578947368421052,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "consensus_hi": {
     "total": 0.06451612903225806,
     "adapted_no_citation": 0.11842105263157894,
     "adapted_citation_dependent": -0.17647058823529413
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 0.834072,
   "retrievalUsd": 0.0,
   "wallSeconds": 207.86,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 18285,
   "tokens": {
    "input": 649,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 20722
   },
   "costByTokenKind": {
    "input": 0.005192,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.82888
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 1,
    "cannotJudge": 0,
    "unresolved": 1
   }
  },
  {
   "taskId": "rr-60543e",
   "system": "plain_tools",
   "block": 13,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.010752688172043012,
     "adapted_no_citation": 0.05263157894736842,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "astra": {
     "total": 0.0,
     "adapted_no_citation": 0.039473684210526314,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "two_judge_mean": {
     "total": 0.005376344086021506,
     "adapted_no_citation": 0.046052631578947366,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "consensus_lo": {
     "total": -0.03225806451612903,
     "adapted_no_citation": 0.0,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "consensus_hi": {
     "total": 0.043010752688172046,
     "adapted_no_citation": 0.09210526315789473,
     "adapted_citation_dependent": -0.17647058823529413
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 5.044466,
   "retrievalUsd": 0.317,
   "wallSeconds": 527.44,
   "modelCalls": 20,
   "searchCalls": 38,
   "fetchCalls": 52,
   "reportChars": 22011,
   "tokens": {
    "input": 42,
    "cacheWrite": 287563,
    "cacheRead": 2809099,
    "output": 40167
   },
   "costByTokenKind": {
    "input": 0.000336,
    "cacheWrite": 2.87563,
    "cacheRead": 0.5618198000000001,
    "output": 1.60668
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-60543e",
   "system": "gpt_researcher",
   "block": 13,
   "status": "completed",
   "scores": {
    "jev": {
     "total": -0.03225806451612903,
     "adapted_no_citation": 0.0,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "astra": {
     "total": -0.03225806451612903,
     "adapted_no_citation": 0.0,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "two_judge_mean": {
     "total": -0.03225806451612903,
     "adapted_no_citation": 0.0,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "consensus_lo": {
     "total": -0.03225806451612903,
     "adapted_no_citation": 0.0,
     "adapted_citation_dependent": -0.17647058823529413
    },
    "consensus_hi": {
     "total": -0.03225806451612903,
     "adapted_no_citation": 0.0,
     "adapted_citation_dependent": -0.17647058823529413
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 6.445152,
   "retrievalUsd": 0.552,
   "wallSeconds": 422.53,
   "modelCalls": 33,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 18744,
   "tokens": {
    "input": 510609,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 59007
   },
   "costByTokenKind": {
    "input": 4.084872,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.36028
   },
   "outputLimitStops": 1,
   "refusalStops": 0,
   "judgments": {
    "expected": 28,
    "disagree": 0,
    "cannotJudge": 0,
    "unresolved": 0
   }
  },
  {
   "taskId": "rr-60542a",
   "system": "closed_book",
   "block": 14,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.7191011235955056,
     "adapted_no_citation": 0.7126436781609196,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.6966292134831461,
     "adapted_no_citation": 0.6896551724137931,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.7078651685393258,
     "adapted_no_citation": 0.7011494252873564,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.651685393258427,
     "adapted_no_citation": 0.6436781609195402,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.7640449438202247,
     "adapted_no_citation": 0.7586206896551724,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 1
   },
   "modelUsd": 0.357184,
   "retrievalUsd": 0.0,
   "wallSeconds": 93.44,
   "modelCalls": 1,
   "searchCalls": 0,
   "fetchCalls": 0,
   "reportChars": 16615,
   "tokens": {
    "input": 293,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 8871
   },
   "costByTokenKind": {
    "input": 0.002344,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 0.35484
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 27,
    "disagree": 3,
    "cannotJudge": 0,
    "unresolved": 3
   }
  },
  {
   "taskId": "rr-60542a",
   "system": "gpt_researcher",
   "block": 14,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.42696629213483145,
     "adapted_no_citation": 0.4367816091954023,
     "adapted_citation_dependent": 0.0
    },
    "astra": {
     "total": 0.5056179775280899,
     "adapted_no_citation": 0.5172413793103449,
     "adapted_citation_dependent": 0.0
    },
    "two_judge_mean": {
     "total": 0.4662921348314607,
     "adapted_no_citation": 0.4770114942528736,
     "adapted_citation_dependent": 0.0
    },
    "consensus_lo": {
     "total": 0.42696629213483145,
     "adapted_no_citation": 0.4367816091954023,
     "adapted_citation_dependent": 0.0
    },
    "consensus_hi": {
     "total": 0.5056179775280899,
     "adapted_no_citation": 0.5172413793103449,
     "adapted_citation_dependent": 0.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 6.93196,
   "retrievalUsd": 0.552,
   "wallSeconds": 443.79,
   "modelCalls": 33,
   "searchCalls": 46,
   "fetchCalls": 230,
   "reportChars": 29446,
   "tokens": {
    "input": 571560,
    "cacheWrite": 0,
    "cacheRead": 0,
    "output": 58987
   },
   "costByTokenKind": {
    "input": 4.57248,
    "cacheWrite": 0.0,
    "cacheRead": 0.0,
    "output": 2.35948
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 27,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  },
  {
   "taskId": "rr-60542a",
   "system": "deepagents",
   "block": 14,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.6853932584269663,
     "adapted_no_citation": 0.6781609195402298,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.6853932584269663,
     "adapted_no_citation": 0.6781609195402298,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.6853932584269663,
     "adapted_no_citation": 0.6781609195402298,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.6853932584269663,
     "adapted_no_citation": 0.6781609195402298,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.6853932584269663,
     "adapted_no_citation": 0.6781609195402298,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 0,
    "astra": 0
   },
   "modelUsd": 6.990069,
   "retrievalUsd": 0.304,
   "wallSeconds": 683.7,
   "modelCalls": 30,
   "searchCalls": 38,
   "fetchCalls": 38,
   "reportChars": 29979,
   "tokens": {
    "input": 76,
    "cacheWrite": 275635,
    "cacheRead": 651156,
    "output": 102572
   },
   "costByTokenKind": {
    "input": 0.000608,
    "cacheWrite": 2.75635,
    "cacheRead": 0.13023120000000002,
    "output": 4.10288
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 27,
    "disagree": 0,
    "cannotJudge": 0,
    "unresolved": 0
   }
  },
  {
   "taskId": "rr-60542a",
   "system": "plain_tools",
   "block": 14,
   "status": "completed",
   "scores": {
    "jev": {
     "total": 0.43820224719101125,
     "adapted_no_citation": 0.42528735632183906,
     "adapted_citation_dependent": 1.0
    },
    "astra": {
     "total": 0.5168539325842697,
     "adapted_no_citation": 0.5057471264367817,
     "adapted_citation_dependent": 1.0
    },
    "two_judge_mean": {
     "total": 0.4775280898876405,
     "adapted_no_citation": 0.4655172413793104,
     "adapted_citation_dependent": 1.0
    },
    "consensus_lo": {
     "total": 0.43820224719101125,
     "adapted_no_citation": 0.42528735632183906,
     "adapted_citation_dependent": 1.0
    },
    "consensus_hi": {
     "total": 0.5168539325842697,
     "adapted_no_citation": 0.5057471264367817,
     "adapted_citation_dependent": 1.0
    }
   },
   "errorsPresent": {
    "jev": 1,
    "astra": 1
   },
   "modelUsd": 2.174333,
   "retrievalUsd": 0.214,
   "wallSeconds": 232.27,
   "modelCalls": 11,
   "searchCalls": 27,
   "fetchCalls": 25,
   "reportChars": 14596,
   "tokens": {
    "input": 24,
    "cacheWrite": 143462,
    "cacheRead": 716607,
    "output": 14905
   },
   "costByTokenKind": {
    "input": 0.000192,
    "cacheWrite": 1.43462,
    "cacheRead": 0.1433214,
    "output": 0.5962
   },
   "outputLimitStops": 0,
   "refusalStops": 0,
   "judgments": {
    "expected": 27,
    "disagree": 2,
    "cannotJudge": 0,
    "unresolved": 2
   }
  }
 ],
 "aiReview": {
  "reviewer": "Codex",
  "reviewType": "AI-assisted, not human adjudication",
  "labels": {
   "cannot_judge": 4,
   "met": 35,
   "not_met": 21
  },
  "labelsApplied": 56,
  "queued": {
   "unresolved": 40,
   "agreements": 20
  },
  "alignment": {
   "agreementsSameAsBothJudges": 20,
   "agreementsDiffering": 0,
   "sidedWithAstra": 28,
   "sidedWithJev": 7,
   "otherUnresolved": 5
  },
  "bounds": {
   "closed_book": {
    "headline": {
     "lo": 0.4977292670217695,
     "hi": 0.6090134263862924
    },
    "withAiPass": {
     "lo": 0.5160999176589223,
     "hi": 0.5995337975036784
    }
   },
   "plain_tools": {
    "headline": {
     "lo": 0.4588316130701298,
     "hi": 0.581119824572117
    },
    "withAiPass": {
     "lo": 0.48056707616218236,
     "hi": 0.5451505385896306
    }
   },
   "gpt_researcher": {
    "headline": {
     "lo": 0.35307072512758597,
     "hi": 0.5066145904252444
    },
    "withAiPass": {
     "lo": 0.3706907710495187,
     "hi": 0.4820626228589628
    }
   },
   "deepagents": {
    "headline": {
     "lo": 0.5134292879928983,
     "hi": 0.6071025740867197
    },
    "withAiPass": {
     "lo": 0.5362705254567773,
     "hi": 0.5945741647163009
    }
   }
  }
 },
 "validation": {
  "heldout": {
   "reports": 30,
   "points": 96,
   "judgesDisagree": 16,
   "astra": {
    "n": 96,
    "accuracy": 0.8854166666666666,
    "balanced_accuracy": 0.875,
    "false_pass": 6,
    "false_fail": 5,
    "unresolved": 0
   },
   "jev": {
    "n": 96,
    "accuracy": 0.8645833333333334,
    "balanced_accuracy": 0.875,
    "false_pass": 10,
    "false_fail": 3,
    "unresolved": 0
   },
   "whenJudgesAgree": {
    "n": 80,
    "accuracy": 0.95,
    "balanced_accuracy": 0.9529914529914529,
    "false_pass": 3,
    "false_fail": 1,
    "unresolved": 0
   }
  },
  "diagnostics": {
   "astra": {
    "cases": 15,
    "correct": 15
   },
   "jev": {
    "cases": 15,
    "correct": 15
   }
  }
 },
 "costsApart": {
  "revisions": [
   {
    "revision": 1,
    "amendment": "amendments/0001-main-12-questions.md",
    "tag": "freeze-r1-a1",
    "commit": "8d85ba0275ff7bc8bbc0ec6e5895e4c49bb202a8",
    "runsOutsideRetainedSet": 12,
    "outsideRetainedSetUsd": 35.4649,
    "operatorStoppedRuns": 2,
    "operatorStoppedUsd": 2.713,
    "fundingInterruptedRuns": 0,
    "fundingInterruptedUsd": 0.0
   },
   {
    "revision": 2,
    "amendment": "amendments/0002-ceiling-and-interrupted-runs.md",
    "tag": "freeze-r1-a2",
    "commit": "0c31d90993bc626558faf5087eb70b0c052f4588",
    "runsOutsideRetainedSet": 0,
    "outsideRetainedSetUsd": 0.0,
    "operatorStoppedRuns": 0,
    "operatorStoppedUsd": 0.0,
    "fundingInterruptedRuns": 8,
    "fundingInterruptedUsd": 7.174300000000001
   }
  ],
  "ledgers": {
   "anthropic": {
    "committedUsd": 341.04,
    "ceilingUsd": 355.0
   },
   "exa": {
    "committedUsd": 17.96,
    "ceilingUsd": 45.0
   },
   "jev": {
    "committedUsd": 0.4,
    "ceilingUsd": 10.0
   }
  }
 },
 "verify": {
  "pass": 35,
  "warn": 0,
  "fail": 0
 },
 "sources": {
  "repositoryCommit": "c959b776c069b4ff06acea9ceea26c9f1c8ad8a7",
  "repositoryTags": [
   "freeze-r1-a3"
  ],
  "note": "sha256 of each study artifact this snapshot was built from. Per-run traces are checked against results.json but not listed.",
  "files": [
   {
    "path": "configs/study.toml",
    "sha256": "75c2ecd8a431a76eda2d50c184430f945e17cd51a815599a6566e01536449b08"
   },
   {
    "path": "configs/funding_amendments.toml",
    "sha256": "6143906102334a3343d097d0880c342be15150d0f2c4d0597b8800a19beff330"
   },
   {
    "path": "configs/splits.json",
    "sha256": "d6bd7cf16b688acda818f87848c4332fd75b52314b88b3a7e3ceff6dc2b57e83"
   },
   {
    "path": "runs/main/results.json",
    "sha256": "c2b752c712233ff16ec0a8a5820b18c63c4e7f22005e78cc873b1bc0cf73e469"
   },
   {
    "path": "runs/main/sensitivity_ai_review.json",
    "sha256": "6e38fa542bbe26367d13e8f5560212a3c74af4b0fac0b229f39556591be94ad9"
   },
   {
    "path": "runs/main/RESULTS.md",
    "sha256": "808bd42cdd602540850db7c1e02239c2545dbe67b73347a2df992b6a77c37f23"
   },
   {
    "path": "runs/main/SENSITIVITY-ai-review.md",
    "sha256": "6a184bb669d4f6eb75e73ba15ca037eed4d749597b01acd24a82e012d39753fd"
   },
   {
    "path": "docs/PROTOCOL.md",
    "sha256": "d83221c92b53d43063a0847bcaf67534044db5d1775be499ae8e77995ac3ed06"
   },
   {
    "path": "docs/COHORT.md",
    "sha256": "f83a5f3ab8b2bf882b0278e22ed6d16ddb7396dacac5f4fbb18e420c76024e21"
   },
   {
    "path": "amendments/0001-main-12-questions.md",
    "sha256": "a148233e7dd4a3471eb85158f0d6b8e0827fc5984a84a135f49155848d831ba8"
   },
   {
    "path": "amendments/0002-ceiling-and-interrupted-runs.md",
    "sha256": "eb8861384f1c417c38afb580362137a4563eb85ae07170f3d4a8660c7592db90"
   },
   {
    "path": "amendments/0003-ai-review-side-analysis.md",
    "sha256": "9af3124f30a868c109f356ea734b2371a343414b745aa56b0d66a06a82f6f735"
   },
   {
    "path": "logs/main-verify-after-0003.log",
    "sha256": "c5a9bbda226deca8dfa7b26c9993746db84bff1e4f9f6265193c3cb7c9710b04"
   },
   {
    "path": "data/public/tasks.jsonl",
    "sha256": "acf365910691809fefaf1fb30c8a0f430d56a7591d73145ea47f195c4670c9e2"
   },
   {
    "path": "data/eval/rubrics.jsonl",
    "sha256": "1a2a92870c0424262b4a6b540047004d8437e41bb1eb93eb1239222df88d4590"
   },
   {
    "path": "runs/main/grades/jev.jsonl",
    "sha256": "1dc1d5fbe39af6b011f27aca1565c51c845dff89b31b5e6a33f2bd6127ad6905"
   },
   {
    "path": "runs/main/grades/astra.jsonl",
    "sha256": "09925367c808ffad48a430c9d07ebc3966a3982b514392fbbbb7321d3d4545e0"
   },
   {
    "path": "runs/main/review_queue.key.json",
    "sha256": "b42d47ac71fe19865713c937e0d99a5074aae19e1a08f5be05c4fc4bbfeedf29"
   },
   {
    "path": "runs/main/grades/ai_review_labels.json",
    "sha256": "48586ddea61b2ead791c2e9251f127ffc3f8a0b4984c9f72056f272d07f55ca8"
   },
   {
    "path": "runs/_grader_validation/diagnostics/diagnostics.json",
    "sha256": "c5255a3a72dc5ef7ccad769fdf26bece69cc331ef8a1c2788a431ae2e7c2e4e1"
   },
   {
    "path": "runs/_grader_validation/heldout-astra_single/validation.json",
    "sha256": "7c9b234c3e319874b1ed6380835e4ddc1e7e4085bc32cf1c36f407a62c54c0cf"
   },
   {
    "path": "budget/anthropic.jsonl",
    "sha256": "26c56fea9c59c0200fe47d23dd88bc251526a88cf0fd6ff123c0383d318d0499"
   },
   {
    "path": "budget/exa.jsonl",
    "sha256": "bbb7048a0c5b44fe5450e89d4de74b4974cbf8b287654ed1bc3f71a0c81ea8eb"
   },
   {
    "path": "budget/jev.jsonl",
    "sha256": "0c1bbd6b2a804d4c71308f2b6248fcb876f41e3e346a44a389b9fbb5ed1a897a"
   }
  ]
 }
}