{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2606.18686", "kind": "paper", "url": "https://arxiv.org/abs/2606.18686"}], "authors": ["Jaeho Lee", "Nick Merrill", "Ezra Karger"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2606.18686", "source_url": "https://arxiv.org/abs/2606.18686"}, "categories": ["General AI", "Language & Knowledge", "cs.AI"], "description": {"en": "A simulated-world forecasting benchmark built on Freeciv game rollouts. Evaluates probabilistic reasoning by scoring forecasts about hidden future game states, with continuous or binary questions, paired intervention worlds, and artifacts for release."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2606.18686", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2606.18686", "title": "ForecastBench-Sim"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2606.18686", "modality": null, "name": "ForecastBench-Sim", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_forecastbench-sim_51b8b6fa", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "cf07fab3ec72ff1e08623225845ffb737ae725776cee8d0243f57c9dbff31caf", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2606.18686"}, "publisher": null, "released": "2026-06-17", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2606.18686", "source_url": "https://arxiv.org/abs/2606.18686"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2606-18686", "source": "claire_radar", "source_benchmark_id": "2606.18686", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2606.18686", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "unavailable"}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "13c680ad7fb6de6d31f24bf5a8af02ae0f341dbae3844150f458bfd8712ba6f5", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "A simulated-world forecasting benchmark built on Freeciv game rollouts. Evaluates probabilistic reasoning by scoring forecasts about hidden future game states, with continuous or binary questions, paired intervention worlds, and artifacts for release.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["exact coined title identity tied to benchmark evidence", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "We introduce ForecastBench-Sim, a simulated-world forecasting benchmark built on game rollouts from Freeciv, a turn-based strategy game modelled on the Civilization series."}, "familyId": "bmf_6ae3da819bfa", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_forecastbench-sim_51b8b6fa", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2606.18686", "paper": "https://arxiv.org/abs/2606.18686", "pdf": "https://arxiv.org/pdf/2606.18686", "project": null, "report": "https://arxiv.org/abs/2606.18686"}, "metrics": [], "motivation": "Forecasting benchmarks for general-purpose AI systems usually inherit the constraints of the real world: outcomes resolve slowly, tail events are rare, and counterfactual questions are difficult to score.", "name": "ForecastBench-Sim", "oneLine": "A simulated-world forecasting benchmark built on Freeciv game rollouts. Evaluates probabilistic reasoning by scoring forecasts about hidden future game states, with continuous or binary questions, paired intervention worlds, and artifacts for release.", "paperTitle": "ForecastBench-Sim: A Simulated-World Forecasting Benchmark", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2606.18686", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {}, "readiness": "Paper only", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-06-17", "source": {"authors": ["Jaeho Lee and Nick Merrill and Ezra Karger"], "categories": ["cs.AI", "cs.CL", "cs.LG"], "id": "2606.18686", "title": "ForecastBench-Sim: A Simulated-World Forecasting Benchmark", "type": "arxiv", "url": "https://arxiv.org/abs/2606.18686"}, "sourceUpdatedAt": "2026-06-17T04:52:41Z", "topics": ["cs.AI"], "venueAttempts": [], "whyItMatters": "Addresses the evaluation gap of slow real-world resolution and rare tail events by providing controllable, immediately resolvable forecasting tasks, enabling rigorous study of AI probabilistic reasoning under dynamic states."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
