{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2606.24530", "kind": "paper", "url": "https://arxiv.org/abs/2606.24530"}, {"id": "gh:frontisai/naturebench", "kind": "repo", "url": "https://github.com/FrontisAI/NatureBench"}], "authors": ["Yuru Wang", "Lejun Cheng", "Yuxin Zuo", "Sihang Zeng", "Bingxiang He", "Che Jiang", "Junlin Yang", "Yuchong Wang", "Kaikai Zhao", "Weifeng Huang", "Kai Tian", "Zhenzhao Yuan", "Jincheng Zhong", "Weizhi Wang", "Ning Ding", "Bowen Zhou", "Kaiyan Zhang"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2606.24530", "source_url": "https://arxiv.org/abs/2606.24530"}, "categories": ["General AI", "Language & Knowledge", "cs.CL"], "description": {"en": "NatureBench evaluates AI coding agents on 90 tasks distilled from Nature-family publications across 6 scientific domains, scoring against each paper's reported state of the art."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2606.24530", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2606.24530", "title": "NatureBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2606.24530", "modality": null, "name": "NatureBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_naturebench_ffd14b37", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "1e833e9bbba527d1f839c6e0e614fca1e3118e353641823996bdd6c8a6714c86", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2606.24530"}, "publisher": null, "released": "2026-06-23", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2606.24530", "source_url": "https://arxiv.org/abs/2606.24530"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2606-24530", "source": "claire_radar", "source_benchmark_id": "2606.24530", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/FrontisAI/NatureBench", "githubScope": "benchmark_repo", "githubStars": 129, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 67, "hfPaperUrl": "https://huggingface.co/papers/2606.24530", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-10-04", "state": "stale"}}, "status": "fresh"}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "17b5881db23bbb713bc8a2955bc962a45b051e371c9b4ea34540417320ecdf94", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "NatureBench evaluates AI coding agents on 90 tasks distilled from Nature-family publications across 6 scientific domains, scoring against each paper's reported state of the art.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "We introduce NatureBench, a cross-discipline benchmark of 90 tasks distilled from peer-reviewed Nature-family publications, designed to evaluate whether AI coding agents can move beyond reproduction toward discovery on real scientific problems."}, "familyId": "bmf_d35b766e0cf9", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "existing-code-link", "linkedUrl": "https://github.com/FrontisAI/NatureBench", "url": "https://github.com/FrontisAI/NatureBench"}, {"basis": "existing-metric-association", "linkedUrl": "https://github.com/FrontisAI/NatureBench", "url": "https://huggingface.co/papers/2606.24530"}, {"basis": "linked-from-report", "linkedUrl": "https://github.com/FrontisAI/NatureBench", "url": "https://arxiv.org/abs/2606.24530"}], "scope": "benchmark_repo", "url": "https://github.com/FrontisAI/NatureBench", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_naturebench_ffd14b37", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": "https://github.com/FrontisAI/NatureBench", "data": null, "hfPaper": "https://huggingface.co/papers/2606.24530", "paper": "https://arxiv.org/abs/2606.24530", "pdf": "https://arxiv.org/pdf/2606.24530", "project": null, "report": "https://arxiv.org/abs/2606.24530"}, "metrics": [], "motivation": "We introduce NatureBench, a cross-discipline benchmark of 90 tasks distilled from peer-reviewed Nature-family publications, designed to evaluate whether AI coding agents can move beyond reproduction toward discovery on real scientific problems.", "name": "NatureBench", "oneLine": "NatureBench evaluates AI coding agents on 90 tasks distilled from Nature-family publications across 6 scientific domains, scoring against each paper's reported state of the art.", "paperTitle": "NatureBench: Can Coding Agents Match the Published SOTA of Nature-Family Papers?", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2606.24530", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "FrontisAI", "organizationType": "company-research-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/FrontisAI/NatureBench"}], "ranking": {}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-06-23", "source": {"authors": ["Yuru Wang", "Lejun Cheng", "Yuxin Zuo", "Sihang Zeng", "Bingxiang He", "Che Jiang", "Junlin Yang", "Yuchong Wang", "Kaikai Zhao", "Weifeng Huang", "Kai Tian", "Zhenzhao Yuan", "Jincheng Zhong", "Weizhi Wang", "Ning Ding", "Bowen Zhou", "Kaiyan Zhang"], "categories": ["cs.CL"], "id": "2606.24530", "title": "NatureBench: Can Coding Agents Match the Published SOTA of Nature-Family Papers?", "type": "arxiv", "url": "https://arxiv.org/abs/2606.24530"}, "sourceUpdatedAt": "2026-07-06T16:56:53Z", "topics": ["cs.CL"], "venueAttempts": [], "whyItMatters": "Provides a standardized environment and public leaderboard to measure whether coding agents can achieve discovery-level performance on real scientific problems, addressing environment-fragmentation issues in prior benchmarks."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
