{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2605.27957", "kind": "paper", "url": "https://arxiv.org/abs/2605.27957"}, {"id": "gh:tamuchen18/disasterbench_open", "kind": "repo", "url": "https://github.com/TamuChen18/DisasterBench_Open"}], "authors": ["Zhitong Chen", "Kai Yin", "Weifeng Zhang", "Zhiyuan Wang", "Xiangjue Dong", "Chengkai Liu", "Zhewei Liu", "Yiming Xiao", "Ali Mostafavi", "James Caverlee"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2605.27957", "source_url": "https://arxiv.org/abs/2605.27957"}, "categories": ["Agents", "General AI", "Language & Knowledge", "Planning"], "description": {"en": "DisasterBench is a benchmark for evaluating structured multi-agent planning over disaster-response tools. It includes 233 expert-verified tasks, 26 agents, 81 typed compatibility edges, and 5 planning paradigms. It uses First-Point-of-Failure (FPoF) for step-level failure attribution."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2605.27957", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2605.27957", "title": "DisasterBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2605.27957", "modality": null, "name": "DisasterBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_disasterbench_d2342204", "origin_source": "arxiv", "recognition_confidence": "0.95", "record_sha256": "69020d26502f3dd4dfbf5962848b8faaefcf0733012b6ec7075b940ec1042887", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2605.27957"}, "publisher": null, "released": "2026-05-27", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2605.27957", "source_url": "https://arxiv.org/abs/2605.27957"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2605-27957", "source": "claire_radar", "source_benchmark_id": "2605.27957", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/TamuChen18/DisasterBench_Open", "githubScope": "benchmark_repo", "githubStars": 2, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2605.27957", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "fresh"}, "capabilities": ["Planning"], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "f07ac4e1004cd46c3a46982e0796b8416bcf07bc68373ac843ced15fc3ba003d", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "DisasterBench is a benchmark for evaluating structured multi-agent planning over disaster-response tools. It includes 233 expert-verified tasks, 26 agents, 81 typed compatibility edges, and 5 planning paradigms. It uses First-Point-of-Failure (FPoF) for step-level failure attribution.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "public artifact URL"], "snippet": "We introduce DisasterBench, a benchmark for evaluating structured multi-agent planning over semantically similar but operationally distinct disaster-response tools."}, "familyId": "bmf_70b465e708e7", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "existing-code-link", "linkedUrl": "https://github.com/TamuChen18/DisasterBench_Open", "url": "https://github.com/TamuChen18/DisasterBench_Open"}, {"basis": "existing-metric-association", "linkedUrl": "https://github.com/TamuChen18/DisasterBench_Open", "url": "https://huggingface.co/papers/2605.27957"}, {"basis": "linked-from-report", "linkedUrl": "https://github.com/TamuChen18/DisasterBench_Open", "url": "https://arxiv.org/abs/2605.27957"}], "scope": "benchmark_repo", "url": "https://github.com/TamuChen18/DisasterBench_Open", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_disasterbench_d2342204", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": "https://github.com/TamuChen18/DisasterBench_Open", "data": null, "hfPaper": "https://huggingface.co/papers/2605.27957", "paper": "https://arxiv.org/abs/2605.27957", "pdf": "https://arxiv.org/pdf/2605.27957", "project": null, "report": "https://arxiv.org/abs/2605.27957"}, "metrics": [], "motivation": "Disasters cause severe societal impacts, demanding rapid coordination of heterogeneous AI tools, from satellite analysis to flood prediction and damage assessment, into coherent multi-step workflows.", "name": "DisasterBench", "oneLine": "DisasterBench is a benchmark for evaluating structured multi-agent planning over disaster-response tools. It includes 233 expert-verified tasks, 26 agents, 81 typed compatibility edges, and 5 planning paradigms. It uses First-Point-of-Failure (FPoF) for step-level failure attribution.", "paperTitle": "DisasterBench: Benchmarking LLM Planning under Typed Tool Interface Constraints", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2605.27957", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "DisasterBench Team", "organizationType": "academic-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/TamuChen18/DisasterBench_Open"}], "ranking": {}, "readiness": "Runnable", "recognitionConfidence": 0.95, "relation": "introduces", "releasedAt": "2026-05-27", "source": {"authors": ["Zhitong Chen", "Kai Yin", "Weifeng Zhang", "Zhiyuan Wang", "Xiangjue Dong", "Chengkai Liu", "Zhewei Liu", "Yiming Xiao", "Ali Mostafavi", "James Caverlee"], "categories": ["cs.CL"], "id": "2605.27957", "title": "DisasterBench: Benchmarking LLM Planning under Typed Tool Interface Constraints", "type": "arxiv", "url": "https://arxiv.org/abs/2605.27957"}, "sourceUpdatedAt": "2026-05-27T04:50:23Z", "topics": ["Agents"], "venueAttempts": [], "whyItMatters": "Disaster response requires orchestrating heterogeneous AI tools into executable workflows. DisasterBench tests grounding under typed interface constraints, highlighting gaps between semantic reasoning and execution consistency, and providing diagnostics for failure attribution."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
