| title | ORBIT: Preserving Foundational Language Capabilities in GenRetrieval via Origin-Regulated Merging |
|---|---|
| authors | Neha Verma, Nikhil Mehta, Shao-Chuan Wang, Naijing Zhang, Alicia Tsai, Li Wei, Lukasz Heldt, Lichan Hong, Ed Chi, Xinyang Yi |
| date | 2026-05-12 |
| venue | arXiv (preprint, not peer reviewed) |
| id | arXiv:2605.12419 |
| upvotes | 1 (Hugging Face Daily Papers, 2026-11-01) |
| links | abs pdf hf |
| abstract | Despite the rapid advancements in large language model (LLM) development, fine-tuning them for specific tasks often results in the catastrophic forgetting of their general, language-based reasoning abilities. This work investigates and addresses this challenge in the context of the Generative Retrieval (GenRetrieval) task. During GenRetrieval fine-tuning, we find this forgetting occurs rapidly and correlates with the distance between the fine-tuned and original model parameters. Given these observations, we propose ORBIT, a novel approach that actively tracks the distance between fine-tuned and initial model weights, and uses a weight averaging strategy to constrain model drift during GenRetrieval fine-tuning when this inter-model distance exceeds a maximum threshold. Our results show that ORBIT retains substantial text and retrieval performance by outperforming both common continual learning baselines and related regularization methods that also employ weight averaging. |
cite
@misc{verma2026orbit,
title={ORBIT: Preserving Foundational Language Capabilities in GenRetrieval via Origin-Regulated Merging},
author={Neha Verma and Nikhil Mehta and Shao-Chuan Wang and Naijing Zhang and Alicia Tsai and Li Wei and Lukasz Heldt and Lichan Hong and Ed Chi and Xinyang Yi},
year={2026},
eprint={2605.12419},
archivePrefix={arXiv},
url={https://arxiv.org/abs/2605.12419}
}