titleZeroVLN: Learning to Navigate without Supervised Fine-Tuning of the VLM Brain
authorsZixuan Huang, Yang Zhou, Guli Zhang, Sunzhu Li, Hongyan Xie, Zehao Chen, Fuzhen Zhuang, Kaixuan Wang, Xiaozhi Chen, Yikun Ban, Deqing Wang
date2026-11-19
venuearXiv (preprint, not peer reviewed)
idarXiv:2611.26414
affiliationZYT AI Team
upvotes21 (Hugging Face Daily Papers, 2026-11-21)
linksabs pdf hf code
cite
abstractRepresentative vision-language navigation (VLN) methods typically couple a vision-language model (VLM) brain with a learnable action head and fine-tune the VLM brain on navigation-domain data to predict geometric goals. However, such adaptation can substantially alter base VLM weights and degrade general visual capabilities. We introduce ZeroVLN, a slow-fast navigation framework that achieves effective navigation with zero navigation-domain SFT of the VLM brain. Rather than teaching the brain to predict geometric goals, we equip it with a semantic tool-calling interface for conveying movement decisions to an action expert in natural language. We first adapt the action expert to carry out these movement requests, then jointly improve the brain's movement decisions and the action expert's continuous execution through closed-loop reinforcement learning (RL). Trained on the same navigation episodes, ZeroVLN substantially outperforms representative SFT-based hierarchical VLN methods on standard VLN-CE (continuous environments) benchmarks, while inducing only minor changes to the brain's weights and largely preserving its general capabilities. To our knowledge, ZeroVLN is the first slow-fast VLN framework with zero navigation-domain SFT of the VLM brain, and the first to jointly optimize its slow and fast policies through closed-loop agentic RL in VLN-CE.

bibtex

@misc{huang2026zerovln,
  title={ZeroVLN: Learning to Navigate without Supervised Fine-Tuning of the VLM Brain},
  author={Zixuan Huang and Yang Zhou and Guli Zhang and Sunzhu Li and Hongyan Xie and Zehao Chen and Fuzhen Zhuang and Kaixuan Wang and Xiaozhi Chen and Yikun Ban and Deqing Wang},
  year={2026},
  eprint={2611.26414},
  archivePrefix={arXiv},
  url={https://arxiv.org/abs/2611.26414}
}