titleSpatialScout: Can VLM Agents Actively Explore and Map 3D Scenes?
authorsZhaoxi Zhang, Hao Liang, Linxuan Wu, Yiming Wu, Yifu Yuan, Kun Liang, Chenming Tang, Meiyi Qiang, Wentao Zhang, Weikang Li
date2026-10-19
venuearXiv (preprint, not peer reviewed)
idarXiv:2610.26659
affiliationPeking University
upvotes120 (Hugging Face Daily Papers, 2026-11-01)
linksabs pdf hf code
abstractSpatial intelligence requires agents to actively acquire visual evidence and integrate observations into a coherent representation of the three-dimensional world. Yet spatial question answering offers limited evidence of whether models can construct accurate metric maps through autonomous exploration. We introduce SpatialScout, a benchmark for evaluating vision-language model (VLM) agents on active object-level 3D mapping. SpatialScout comprises 80 diverse indoor scenes containing 3,140 target objects. Given an object inventory, RGB observations, camera calibration, and camera poses, agents control a camera to explore each scene and estimate object centers in a shared coordinate frame. The task requires coordinating viewpoint selection, cross-view association, geometric estimation, and map revision under bounded interaction budgets. Evaluation jointly measures localization accuracy and target coverage through object-scale overlap scores that penalize missing predictions. Experiments with frontier VLMs reveal substantial limitations: the strongest evaluated model, Claude Opus 5.5, achieves a mean overlap score of 41.71%, with 47.23% of targets exceeding an intersection-over-union (IoU) threshold of 0.5 and only 3.25% exceeding 0.9. These results reveal a substantial gap between recovering coarse object layouts and constructing complete, precise metric maps. SpatialScout provides a quantitative testbed for developing agents that build reliable 3D spatial representations through active perception. Code is available at https://github.com/JesseZZZZZ/SpatialScout.

cite

@misc{zhang2026spatialscout,
  title={SpatialScout: Can VLM Agents Actively Explore and Map 3D Scenes?},
  author={Zhaoxi Zhang and Hao Liang and Linxuan Wu and Yiming Wu and Yifu Yuan and Kun Liang and Chenming Tang and Meiyi Qiang and Wentao Zhang and Weikang Li},
  year={2026},
  eprint={2610.26659},
  archivePrefix={arXiv},
  url={https://arxiv.org/abs/2610.26659}
}