| title | Agentic RAG Evaluation: Budget Allocation Across Questions, Trajectories, and Reads |
|---|---|
| authors | Jingjie Ning, Xueqi Li, Yibo Kong |
| date | 2026-10-04 |
| venue | arXiv (preprint, not peer reviewed) |
| id | arXiv:2610.05034 |
| affiliation | Carnegie Mellon University |
| upvotes | 10 (Hugging Face Daily Papers, 2026-11-01) |
| links | abs pdf hf |
| abstract | Evaluation budgets in agentic retrieval-augmented generation span questions, search trajectories, and repeated answers. We measure allocation precision, reading efficiency, and cost boundaries using a retrieval-feedback comparison on HotpotQA and MuSiQue. At 34.14--34.39M model tokens, broader question coverage lowers standard error by 33\% versus five reads and 12.6\% versus three trajectories. Archived nested and Q-only forecasts predict these allocations within 4.0\% and 3.5\%, respectively. Depth subsets establish no clear forecasting advantage beyond the two-trajectory audit. One-read variance penalties relative to the fitted optimum at the same token budget are 0--9.9\%, with substantial Pro uncertainty. Under recorded model fees, more questions beat more trajectories at search prices of \$0--1 per 1,000 requests; question-versus-read fee rankings remain unresolved. Temperature zero cuts answer disagreement from 14.3\% to 3.4\% while comparison precision stays similar. \par\medskip\noindentKeywords: Agentic RAG; Evaluation budget; Generalizability theory; Repeated sampling. |
cite
@misc{ning2026agentic,
title={Agentic RAG Evaluation: Budget Allocation Across Questions, Trajectories, and Reads},
author={Jingjie Ning and Xueqi Li and Yibo Kong},
year={2026},
eprint={2610.05034},
archivePrefix={arXiv},
url={https://arxiv.org/abs/2610.05034}
}