{"record": {"aliases": ["PhAIL: A Real-Robot VLA Benchmark and Distributional Methodology"], "artifacts": [{"id": "arxiv:2605.29710", "kind": "paper", "url": "https://arxiv.org/abs/2605.29710"}, {"id": "gh:positronic-robotics/phail-paper", "kind": "repo", "url": "https://github.com/Positronic-Robotics/phail-paper"}, {"kind": "website", "url": "https://phail.ai"}], "authors": ["Sergey Arkhangelskiy"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2605.29710", "source_url": "https://arxiv.org/abs/2605.29710"}, "categories": ["Robotics", "Robotics & Embodied AI"], "description": {"en": "A real-robot leaderboard for vision-language-action policies on a Franka FR3. It scores policies with Human-Relative Throughput computed from time-to-success distributions and uses Kolmogorov-Smirnov significance tests."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2605.29710", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2605.29710", "title": "PhAIL (Physical AI Leaderboard)"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["Positron Systems (United States)"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2605.29710", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2605.29710", "source_url": "https://openalex.org/W7162776864"}, "key": "claire-radar:2605.29710", "modality": null, "name": "PhAIL (Physical AI Leaderboard)", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"admission_policy_version": "2026-09-09.1", "confidence": "Medium", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-09-25T04:40:22Z", "data_status": "primary-source-indexed", "display_eligible": "true", "origin_record_id": "bm_phail-a-real-robot-vla-benchmark-and-distr_f8299998", "origin_source": "arxiv", "recognition_confidence": "0.7", "record_sha256": "07087600fd14b205b4a99c38117ab3eced1e52211454aa0c9355475816328a1d", "relation": "introduces", "review_model": "claude-code-default", "review_state": "ai-reviewed", "reviewed_at": "2026-09-24T16:49:37.728303Z", "source_url": "https://arxiv.org/abs/2605.29710"}, "publisher": null, "released": "2026-05-28", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2605.29710", "source_url": "https://arxiv.org/abs/2605.29710"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2605-29710", "source": "claire_radar", "source_benchmark_id": "2605.29710", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": ["PhAIL: A Real-Robot VLA Benchmark and Distributional Methodology"], "annotation": "Unknown", "applicationDomains": ["Robotics & Embodied AI"], "area": "Robotics & Embodied AI", "attentionForecast": {"confidence": "Medium", "horizon": "7d", "reason": "A public leaderboard for real-robot VLA evaluation with a released dataset and pipeline is likely to draw interest from robotics evaluation researchers.", "score": 55}, "capabilities": [], "confidence": "Medium", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-09-24T16:49:37.728303Z", "inputHash": "3e27610bcfede37b9f6fd3590dbc6b544baf64572c3f3a78123b7e1b2c4085c3", "model": "claude-code-default"}, "curation": {"admissionPolicyVersion": "2026-09-09.1", "canonicalNameEvidence": "We introduce PhAIL (Physical AI Leaderboard, https://phail.ai), an open real-robot benchmark on a Franka FR3", "canonicalNameSource": "abstract", "decisionReason": "The benchmark is formally named and has a defined scalar score (HRT with bootstrap CIs) and a defined significance test. A public leaderboard site, the dataset, and the analysis pipeline are all released.", "evaluationEvidence": {"reuse": "Dataset, analysis pipeline, and paper source: https://phail.ai and https://github.com/Positronic-Robotics/phail-paper", "scoring": "scoring via Human-Relative Throughput (HRT), a dimensionless scalar with bootstrap confidence intervals", "task": "an open real-robot benchmark on a Franka FR3"}, "model": "claude-code-default", "reviewedAt": "2026-09-24T16:49:37.728303Z", "state": "ai-reviewed"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "A real-robot leaderboard for vision-language-action policies on a Franka FR3. It scores policies with Human-Relative Throughput computed from time-to-success distributions and uses Kolmogorov-Smirnov significance tests.", "displayEligible": true, "domainCuration": {"method": "rules-v1", "state": "auto"}, "evaluationMode": "public_reusable", "evidence": {"reasonCodes": ["benchmark term in abstract", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "We introduce PhAIL (Physical AI Leaderboard, https://phail.ai), an open real-robot benchmark on a Franka FR3 (dataset, per-rollout artifacts, and end-to-end reference implementation) of a distributional evaluation methodology: the time-to-success cumulative distribution function (CDF) as the evaluation primitive, with two separated jobs."}, "familyId": "bmf_10dd88a72d57", "firstSeenAt": "2026-08-19", "heat": null, "id": "bm_phail-a-real-robot-vla-benchmark-and-distr_f8299998", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": ["Robotics"], "links": {"code": "https://github.com/Positronic-Robotics/phail-paper", "data": null, "paper": "https://arxiv.org/abs/2605.29710", "pdf": "https://arxiv.org/pdf/2605.29710", "project": "https://phail.ai", "report": "https://arxiv.org/abs/2605.29710"}, "metrics": [], "motivation": "Real-world evaluation of vision-language-action (VLA) policies still rests on binary success rate at a fixed timeout with $N \\le 25$ rollouts per condition, almost always without confidence intervals or paired statistical comparison; these cohort sizes struggle to resolve close comparisons reliably.", "name": "PhAIL (Physical AI Leaderboard)", "oneLine": "A real-robot leaderboard for vision-language-action policies on a Franka FR3. It scores policies with Human-Relative Throughput computed from time-to-success distributions and uses Kolmogorov-Smirnov significance tests.", "paperTitle": "PhAIL: A Real-Robot VLA Benchmark and Distributional Methodology", "primaryDomain": "Robotics & Embodied AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2605.29710", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-08-19T11:05:13.395391Z"}, "publications": [], "publishers": [{"name": "Positronic Robotics", "organizationType": "company-research-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/Positronic-Robotics/phail-paper"}], "readiness": "Runnable", "recognitionConfidence": 0.7, "relation": "introduces", "releasedAt": "2026-05-28", "reviewLifecycle": {"state": "pending"}, "source": {"authors": ["Sergey Arkhangelskiy"], "categories": ["cs.RO"], "id": "2605.29710", "title": "PhAIL: A Real-Robot VLA Benchmark and Distributional Methodology", "type": "arxiv", "url": "https://arxiv.org/abs/2605.29710"}, "sourceUpdatedAt": "2026-05-28T10:10:19Z", "topics": ["Robotics"], "venueAttempts": [], "whyItMatters": "It replaces binary success rates from small rollout counts with a distributional metric that includes confidence intervals and is anchored to human teleoperation. According to the abstract, this separates close VLA comparisons that threshold metrics cannot resolve."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
