{"record": {"aliases": ["PRM-as-a-Judge 1.5: A Toolkit for Robot Process Assessment"], "artifacts": [{"id": "arxiv:2608.14284", "kind": "paper", "url": "https://arxiv.org/abs/2608.14284"}, {"kind": "website", "url": "https://prm-as-a-judge.github.io"}], "authors": ["Yuyang Liu", "Yanqing Shen", "Ruike Chen", "Jifan Zhao", "Yuxuan Tian", "Yichi Zhang", "Tianfeng Long", "Zixuan Yin", "Yipu Wang", "Ziheng Qin", "Wenxing Tan", "Yang Shi", "Mingyu Cao", "Runze Xiao", "Ziqi Wang", "Zhixin Yin", "Shiwei Chu", "Yi-Fan Zhang", "Yao Mu", "Yuheng Ji", "Yihao Wang", "Jun Yan", "Zhongyuan Wang", "Pengwei Wang", "Xiaolong Zheng"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.14284", "source_url": "https://arxiv.org/abs/2608.14284"}, "categories": ["Robot manipulation", "Robotics", "Robotics & Embodied AI"], "description": {"en": "A toolkit that converts robotic rollout videos into progress curves and computes metrics for failure progress, recovery, and execution quality, with a benchmark and evaluation suite for process reward models."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.14284", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.14284", "title": "PRM-as-a-Judge 1.5"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2608.14284", "modality": null, "name": "PRM-as-a-Judge 1.5", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "Medium", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "true", "origin_record_id": "bm_prm-as-a-judge-1-5-a-toolkit-for-robot-pro_42c65f99", "origin_source": "arxiv", "recognition_confidence": "0.7", "record_sha256": "60d8c0015434b08687a750298dda63d2e0cd92eb3a79eb40d7ed7cdb32e564f3", "relation": "introduces", "review_model": "deepseek-v4-pro", "review_state": "ai-reviewed", "reviewed_at": "2026-08-26T06:10:57.123445Z", "source_url": "https://arxiv.org/abs/2608.14284"}, "publisher": null, "released": "2026-08-14", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.14284", "source_url": "https://arxiv.org/abs/2608.14284"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-14284", "source": "claire_radar", "source_benchmark_id": "2608.14284", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": ["PRM-as-a-Judge 1.5: A Toolkit for Robot Process Assessment"], "annotation": "Unknown", "applicationDomains": ["Robotics & Embodied AI"], "area": "Robotics & Embodied AI", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/YuyangLiu2003/PRM-as-a-Judge", "githubScope": "benchmark_repo", "githubStars": 41, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 15, "hfPaperUrl": "https://huggingface.co/papers/2608.14284", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-30", "state": "stale"}}, "status": "fresh"}, "attentionForecast": {"confidence": "Medium", "horizon": "7d", "reason": "The clear project page, toolkit release, and focus on reproducible process assessment for robotics are likely to drive interest from the embodied AI community.", "score": 55}, "capabilities": ["Robot manipulation"], "confidence": "Medium", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-26T06:10:57.123445Z", "inputHash": "e5906a6020650e486a8ca5b62cee08bba5bdeb2a1b876db31b7f2d7707a71259", "model": "deepseek-v4-pro"}, "curation": {"canonicalNameEvidence": "We present PRM-as-a-Judge 1.5, a toolkit for robot process assessment", "canonicalNameSource": "abstract", "decisionReason": "The paper describes a toolkit with a released assessment suite including benchmark, metrics, and visualization tools, and provides a project page, establishing a public reuse path and stable scoring contract.", "model": "deepseek-v4-pro", "reviewedAt": "2026-08-26T06:10:57.123445Z", "state": "ai-reviewed"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "A toolkit that converts robotic rollout videos into progress curves and computes metrics for failure progress, recovery, and execution quality, with a benchmark and evaluation suite for process reward models.", "displayEligible": true, "domainCuration": {"method": "rules-v1", "state": "auto"}, "evaluationMode": "score_submission", "evidence": {"reasonCodes": ["benchmark term in abstract", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "Moreover, we release a user-friendly assessment suite, including the benchmark, metric implementation, and visualization tools, to support reproducible manipulation process evaluation."}, "familyId": "bmf_3a1382624b18", "firstSeenAt": "2026-08-19", "heat": null, "id": "bm_prm-as-a-judge-1-5-a-toolkit-for-robot-pro_42c65f99", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": ["Robotics"], "links": {"code": null, "data": null, "paper": "https://arxiv.org/abs/2608.14284", "pdf": "https://arxiv.org/pdf/2608.14284", "project": "https://prm-as-a-judge.github.io", "report": "https://arxiv.org/abs/2608.14284"}, "metrics": [], "motivation": "Fine-grained robotic evaluation matters for understanding embodied models, going beyond binary success rates and rule-based process scores.", "name": "PRM-as-a-Judge 1.5", "oneLine": "A toolkit that converts robotic rollout videos into progress curves and computes metrics for failure progress, recovery, and execution quality, with a benchmark and evaluation suite for process reward models.", "paperTitle": "PRM-as-a-Judge 1.5: A Toolkit for Robot Process Assessment", "primaryDomain": "Robotics & Embodied AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.14284", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-08-19T11:05:13.395391Z"}, "publications": [], "ranking": {"90d": {"confidence": "Medium", "coverage": 0.7, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.4989117052391329, "value": 41}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.48990148153820845, "value": 15}}, "confidence": "Medium", "coverage": 0.7, "rank": 67, "score": 50}, "method": "attention-ranking-v13", "rank": 67, "score": 50}}, "readiness": "Paper only", "recognitionConfidence": 0.7, "relation": "introduces", "releasedAt": "2026-08-14", "source": {"authors": ["Yuyang Liu", "Yanqing Shen", "Ruike Chen", "Jifan Zhao", "Yuxuan Tian", "Yichi Zhang", "Tianfeng Long", "Zixuan Yin", "Yipu Wang", "Ziheng Qin", "Wenxing Tan", "Yang Shi", "Mingyu Cao", "Runze Xiao", "Ziqi Wang", "Zhixin Yin", "Shiwei Chu", "Yi-Fan Zhang", "Yao Mu", "Yuheng Ji", "Yihao Wang", "Jun Yan", "Zhongyuan Wang", "Pengwei Wang", "Xiaolong Zheng"], "categories": ["cs.RO", "cs.CV"], "id": "2608.14284", "title": "PRM-as-a-Judge 1.5: A Toolkit for Robot Process Assessment", "type": "arxiv", "url": "https://arxiv.org/abs/2608.14284"}, "sourceUpdatedAt": "2026-08-14T13:10:02Z", "topics": ["Robotics"], "venueAttempts": [], "whyItMatters": "Supplies fine-grained process assessment beyond binary success, enabling more transparent and procedural evaluation of embodied models."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
