| title | PRM-as-a-Judge 1.5: A Toolkit for Robot Process Assessment |
|---|---|
| authors | Yuyang Liu, Yanqing Shen, Ruike Chen, Jifan Zhao, Yuxuan Tian, Yichi Zhang, Tianfeng Long, Zixuan Yin, Yipu Wang, Ziheng Qin, Wenxing Tan, Yang Shi, Mingyu Cao, Runze Xiao, Ziqi Wang, Zhixin Yin, Shiwei Chu, Yi-Fan Zhang, Yao Mu, Yuheng Ji, Yihao Wang, Jun Yan, Zhongyuan Wang, Pengwei Wang, Xiaolong Zheng |
| date | 2026-08-14 |
| venue | arXiv (preprint, not peer reviewed) |
| id | arXiv:2608.14284 |
| upvotes | 15 (Hugging Face Daily Papers, 2026-11-01) |
| links | abs pdf hf code |
| abstract | Fine-grained robotic evaluation matters for understanding embodied models, going beyond binary success rates and rule-based process scores. We present PRM-as-a-Judge 1.5, a toolkit for robot process assessment that turns rollout videos into dense progress curves and derives multiple fine metrics. PRM-as-a-Judge 1.5 introduces three metrics, building on version 1.0, that characterize failure-side progress, post-drawdown recovery, and success-side execution quality, helping users understand embodied model capability. Based on the rollout videos from benchmarks, we perform a comprehensive assessment of the embodied models, providing some fine-grained metric results and key findings. We further introduce RoboPulse++ to evaluate the reliability of process reward models (PRM), providing evaluators with a more accurate testing platform. Moreover, we release a user-friendly assessment suite, including the benchmark, metric implementation, and visualization tools, to support reproducible manipulation process evaluation. We call on the community to rethink how robots are evaluated and establish transparent, procedural, and reproducible assessment as a foundation for the next generation of embodied intelligence. |
cite
@misc{liu2026prm,
title={PRM-as-a-Judge 1.5: A Toolkit for Robot Process Assessment},
author={Yuyang Liu and Yanqing Shen and Ruike Chen and Jifan Zhao and Yuxuan Tian and Yichi Zhang and Tianfeng Long and Zixuan Yin and Yipu Wang and Ziheng Qin and Wenxing Tan and Yang Shi and Mingyu Cao and Runze Xiao and Ziqi Wang and Zhixin Yin and Shiwei Chu and Yi-Fan Zhang and Yao Mu and Yuheng Ji and Yihao Wang and Jun Yan and Zhongyuan Wang and Pengwei Wang and Xiaolong Zheng},
year={2026},
eprint={2608.14284},
archivePrefix={arXiv},
url={https://arxiv.org/abs/2608.14284}
}