{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2607.04686", "kind": "paper", "url": "https://arxiv.org/abs/2607.04686"}], "authors": ["Harsh Soni"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2607.04686", "source_url": "https://arxiv.org/abs/2607.04686"}, "categories": ["Cybersecurity", "Finance", "Language & Knowledge", "cs.CL"], "description": {"en": "ToolFailBench evaluates tool-use failures in LLM agents across 1,000 tasks in finance, medicine, law, cybersecurity, and real estate. It labels traces with Tool-Skip, Result-Ignore, Output-Fabrication, and Unnecessary-Tool-Use, using a rule classifier and LLM judges."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2607.04686", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2607.04686", "title": "ToolFailBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["University of California, Berkeley"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2607.04686", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2607.04686", "source_url": "https://openalex.org/W7167627487"}, "key": "claire-radar:2607.04686", "modality": null, "name": "ToolFailBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_toolfailbench_999a8910", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "28a6be355030beb908ca01083318c91b413d483b6c10858e07afaf9968a3058e", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2607.04686"}, "publisher": null, "released": "2026-07-06", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2607.04686", "source_url": "https://arxiv.org/abs/2607.04686"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2607-04686", "source": "claire_radar", "source_benchmark_id": "2607.04686", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Mixed", "applicationDomains": ["Cybersecurity", "Finance"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2607.04686", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "unavailable"}, "capabilities": [], "confidence": "High", "construction": "Aggregate Existing", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "7bd9bd8aa8b7a1e612f0620acdbd7999fe753d02c91af512f02ca6a92f9c527f", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "ToolFailBench evaluates tool-use failures in LLM agents across 1,000 tasks in finance, medicine, law, cybersecurity, and real estate. It labels traces with Tool-Skip, Result-Ignore, Output-Fabrication, and Unnecessary-Tool-Use, using a rule classifier and LLM judges.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "We introduce ToolFailBench, a diagnostic benchmark for measuring tool-use failures across 1,000 tasks in finance, medicine, law, cybersecurity, and real estate."}, "familyId": "bmf_432789540d43", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_toolfailbench_999a8910", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": ["Cybersecurity", "Financial Services"], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2607.04686", "paper": "https://arxiv.org/abs/2607.04686", "pdf": "https://arxiv.org/pdf/2607.04686", "project": null, "report": "https://arxiv.org/abs/2607.04686"}, "metrics": [], "motivation": "Tool calling is central to modern language model agents, but aggregate benchmark scores often hide where tool use fails.", "name": "ToolFailBench", "oneLine": "ToolFailBench evaluates tool-use failures in LLM agents across 1,000 tasks in finance, medicine, law, cybersecurity, and real estate. It labels traces with Tool-Skip, Result-Ignore, Output-Fabrication, and Unnecessary-Tool-Use, using a rule classifier and LLM judges.", "paperTitle": "ToolFailBench: Diagnosing Tool-Use Failures in LLM Agents", "primaryDomain": "Cybersecurity", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2607.04686", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {}, "readiness": "Paper only", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-07-06", "source": {"authors": ["Harsh Soni"], "categories": ["cs.CL", "cs.AI", "cs.SE"], "id": "2607.04686", "title": "ToolFailBench: Diagnosing Tool-Use Failures in LLM Agents", "type": "arxiv", "url": "https://arxiv.org/abs/2607.04686"}, "sourceUpdatedAt": "2026-07-06T05:25:21Z", "topics": ["cs.CL"], "venueAttempts": [], "whyItMatters": "Aggregate accuracy hides distinct failure modes in tool use. This benchmark separates models that fail to call tools from those that call but ignore results, enabling targeted diagnosis of agent reliability."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
