{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2606.22388", "kind": "paper", "url": "https://arxiv.org/abs/2606.22388"}, {"id": "gh:jiayujeff/planbench-xl", "kind": "repo", "url": "https://github.com/JiayuJeff/PlanBench-XL"}], "authors": ["Jiayu Liu", "Qihan Lin", "Cheng Qian", "Rui Wang", "Emre Can Acikgoz", "Xiaocheng Yang", "Jiateng Liu", "Zhenhailong Wang", "Xiusi Chen", "Heng Ji", "Dilek Hakkani-Tür"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2606.22388", "source_url": "https://arxiv.org/abs/2606.22388"}, "categories": ["Agents & Tool Use", "General AI", "Planning", "cs.AI"], "description": {"en": "PlanBench-XL evaluates LLM tool-use agents on long-horizon planning in large-scale tool ecosystems. It includes 327 retail tasks across 1,665 tools, requiring iterative tool retrieval and use. Optional blocker mechanisms inject missing, failing, or distracting tools to test adaptive planning. Scoring is based on final answer accuracy and auxiliary metrics."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2606.22388", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2606.22388", "title": "PlanBench-XL"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2606.22388", "modality": null, "name": "PlanBench-XL", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_planbench-xl_459bd32f", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "4a3b0c5a915e097a33cb1bd9d4de253bb45e8f62bfe893a34fbbcf09ffd549ec", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2606.22388"}, "publisher": null, "released": "2026-06-21", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2606.22388", "source_url": "https://arxiv.org/abs/2606.22388"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2606-22388", "source": "claire_radar", "source_benchmark_id": "2606.22388", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Agents & Tool Use", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/JiayuJeff/PlanBench-XL", "githubScope": "benchmark_repo", "githubStars": 40, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 96, "hfPaperUrl": "https://huggingface.co/papers/2606.22388", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-10-04", "state": "stale"}}, "status": "fresh"}, "capabilities": ["Planning"], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "cbfd3b537693765f431621cd94544bd7036ffce327d95eff0c803eff393bfa90", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "PlanBench-XL evaluates LLM tool-use agents on long-horizon planning in large-scale tool ecosystems. It includes 327 retail tasks across 1,665 tools, requiring iterative tool retrieval and use. Optional blocker mechanisms inject missing, failing, or distracting tools to test adaptive planning. Scoring is based on final answer accuracy and auxiliary metrics.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["exact coined title identity tied to benchmark evidence", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "To address this gap, we introduce PlanBench-XL, an interactive benchmark of 327 retail tasks over 1,665 tools that tests whether agents can iteratively retrieve usable tools, invoke them to uncover intermediate evidence for subsequent calls toward the final goal."}, "familyId": "bmf_023188610684", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "existing-code-link", "linkedUrl": "https://github.com/JiayuJeff/PlanBench-XL", "url": "https://github.com/JiayuJeff/PlanBench-XL"}, {"basis": "existing-metric-association", "linkedUrl": "https://github.com/JiayuJeff/PlanBench-XL", "url": "https://huggingface.co/papers/2606.22388"}], "scope": "benchmark_repo", "url": "https://github.com/JiayuJeff/PlanBench-XL", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_planbench-xl_459bd32f", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": "https://github.com/JiayuJeff/PlanBench-XL", "data": null, "hfPaper": "https://huggingface.co/papers/2606.22388", "paper": "https://arxiv.org/abs/2606.22388", "pdf": "https://arxiv.org/pdf/2606.22388", "project": null, "report": "https://arxiv.org/abs/2606.22388"}, "metrics": [], "motivation": "LLM agents increasingly operate in large tool ecosystems, where real-world tasks require discovering relevant tools, inferring implicit sub-goals, and adapting to dynamic environments over long horizons.", "name": "PlanBench-XL", "oneLine": "PlanBench-XL evaluates LLM tool-use agents on long-horizon planning in large-scale tool ecosystems. It includes 327 retail tasks across 1,665 tools, requiring iterative tool retrieval and use. Optional blocker mechanisms inject missing, failing, or distracting tools to test adaptive planning. Scoring is based on final answer accuracy and auxiliary metrics.", "paperTitle": "PlanBench-XL: Evaluating Long-Horizon Planning of LLM Tool-Use Agents in Large-Scale Tool Ecosystems", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2606.22388", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "PlanBench-XL Team", "organizationType": "academic-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/JiayuJeff/PlanBench-XL"}], "ranking": {}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-06-21", "source": {"authors": ["Jiayu Liu", "Qihan Lin", "Cheng Qian", "Rui Wang", "Emre Can Acikgoz", "Xiaocheng Yang", "Jiateng Liu", "Zhenhailong Wang", "Xiusi Chen", "Heng Ji", "Dilek Hakkani-T\\\"ur"], "categories": ["cs.AI", "cs.CL"], "id": "2606.22388", "title": "PlanBench-XL: Evaluating Long-Horizon Planning of LLM Tool-Use Agents in Large-Scale Tool Ecosystems", "type": "arxiv", "url": "https://arxiv.org/abs/2606.22388"}, "sourceUpdatedAt": "2026-06-21T08:29:12Z", "topics": ["cs.AI"], "venueAttempts": [], "whyItMatters": "Existing benchmarks often assume full tool visibility, which underrepresents real-world agent deployment. PlanBench-XL fills this gap by testing planning under retrieval-limited visibility and tool failures, providing insight into robustness and adaptability of LLM agents in complex tool environments."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
