{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2605.22079", "kind": "paper", "url": "https://arxiv.org/abs/2605.22079"}], "authors": ["Ryo Kanazawa", "Koyo Hidaka", "Teppei Miyamoto", "Takayuki Kato", "Tomoki Ando", "Chenguang Wang", "Dayuan Jiang", "Naofumi Fujita", "Shuhei Saitoh", "Atomu Kondo", "Koki Arakawa", "Daiho Nishioka"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2605.22079", "source_url": "https://arxiv.org/abs/2605.22079"}, "categories": ["General AI", "Language & Knowledge", "cs.CL"], "description": {"en": "A benchmark for generating Information Delivery Specification (IDS) XML from BIM information requirements, with 166 examples in English and Japanese. It evaluates LLMs on formal validity via IDSAuditTool and content fidelity via facet-level macro-F1 against gold IDS files."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2605.22079", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2605.22079", "title": "Ishigaki-IDS-Bench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2605.22079", "modality": null, "name": "Ishigaki-IDS-Bench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "Medium", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_ishigaki-ids-bench_d58a0879", "origin_source": "arxiv", "recognition_confidence": "0.85", "record_sha256": "c625ef553c8288a51673a7b3016ebec9700545dc100337afc18d4e811652a059", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2605.22079"}, "publisher": null, "released": "2026-05-21", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2605.22079", "source_url": "https://arxiv.org/abs/2605.22079"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2605-22079", "source": "claire_radar", "source_benchmark_id": "2605.22079", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 0, "hfPaperUrl": "https://huggingface.co/papers/2605.22079", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}}, "status": "fresh"}, "capabilities": [], "confidence": "Medium", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "c27197e53cf8db2dad459e76339bd492abf8bcdc7e06d7a1555d7d6cf7f0931b", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "A benchmark for generating Information Delivery Specification (IDS) XML from BIM information requirements, with 166 examples in English and Japanese. It evaluates LLMs on formal validity via IDSAuditTool and content fidelity via facet-level macro-F1 against gold IDS files.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence"], "snippet": "We present Ishigaki-IDS-Bench, the first publicly released benchmark for IDS generation from BIM information requirements."}, "familyId": "bmf_4b152674589d", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_ishigaki-ids-bench_d58a0879", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2605.22079", "paper": "https://arxiv.org/abs/2605.22079", "pdf": "https://arxiv.org/pdf/2605.22079", "project": null, "report": "https://arxiv.org/abs/2605.22079"}, "metrics": [], "motivation": "Building Information Modeling (BIM) projects increasingly use Information Delivery Specification (IDS) to formalize information requirements in a machine-checkable XML format.", "name": "Ishigaki-IDS-Bench", "oneLine": "A benchmark for generating Information Delivery Specification (IDS) XML from BIM information requirements, with 166 examples in English and Japanese. It evaluates LLMs on formal validity via IDSAuditTool and content fidelity via facet-level macro-F1 against gold IDS files.", "paperTitle": "Ishigaki-IDS-Bench: A Benchmark for Generating Information Delivery Specification from BIM Information Requirements", "primaryDomain": "General AI", "publication": {"evidence": "5 pages, 2 figures. Accepted at CIKM '26 (35th ACM International Conference on Information and Knowledge Management), Rome, Italy, November 2026; resource track, oral presentation. ACM DOI: 10.1145/3799682.3840206. Benchmark data on Hugging Face and evaluation code on Zenodo", "evidenceLevel": "author-claim", "evidenceUrl": "https://arxiv.org/abs/2605.22079", "source": "arxiv-comments", "status": "acceptance_claimed", "venue": "CIKM '26 (35th ACM International Conference on Information and Knowledge Management), Rome, Italy, November 2026", "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {}, "readiness": "Paper only", "recognitionConfidence": 0.85, "relation": "introduces", "releasedAt": "2026-05-21", "source": {"authors": ["Ryo Kanazawa", "Koyo Hidaka", "Teppei Miyamoto", "Takayuki Kato", "Tomoki Ando", "Chenguang Wang", "Dayuan Jiang", "Naofumi Fujita", "Shuhei Saitoh", "Atomu Kondo", "Koki Arakawa", "and Daiho Nishioka"], "categories": ["cs.CL"], "id": "2605.22079", "title": "Ishigaki-IDS-Bench: A Benchmark for Generating Information Delivery Specification from BIM Information Requirements", "type": "arxiv", "url": "https://arxiv.org/abs/2605.22079"}, "sourceUpdatedAt": "2026-06-07T09:44:40Z", "topics": ["cs.CL"], "venueAttempts": [{"decisionRaw": "5 pages, 2 figures. Accepted at CIKM '26 (35th ACM International Conference on Information and Knowledge Management), Rome, Italy, November 2026; resource track, oral presentation. ACM DOI: 10.1145/3799682.3840206. Benchmark data on Hugging Face and evaluation code on Zenodo", "evidence": [{"level": "author-claim", "observedAt": "2026-10-14T01:46:58.058199Z", "rawValue": "5 pages, 2 figures. Accepted at CIKM '26 (35th ACM International Conference on Information and Knowledge Management), Rome, Italy, November 2026; resource track, oral presentation. ACM DOI: 10.1145/3799682.3840206. Benchmark data on Hugging Face and evaluation code on Zenodo", "sourceType": "arxiv-comments", "sourceUrl": "https://arxiv.org/abs/2605.22079"}], "reviewStatus": "accepted", "venueName": "CIKM '26 (35th ACM International Conference on Information and Knowledge Management), Rome, Italy, November 2026"}], "whyItMatters": "IDS authoring requires domain expertise and vocabulary conformance; this benchmark quantifies LLM performance on a structured generation task where output must satisfy external validation tools, filling a gap for capability assessment in BIM-specific language models."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
