{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2608.17271", "kind": "paper", "url": "https://arxiv.org/abs/2608.17271"}, {"id": "gh:apexin-ai/asi-bench", "kind": "repo", "url": "https://github.com/apexin-ai/ASI-Bench"}, {"id": "hf:apexintelligence-ai/asi-bench-seed31415", "kind": "dataset", "url": "https://huggingface.co/datasets/Apexintelligence-AI/ASI-Bench-seed31415"}, {"kind": "website", "url": "https://asibench.apexin.ai"}], "authors": ["Junwei Zhou", "Zhen Sun", "Binyu Li", "Jiangyu Zhou", "Yuexi Pan", "Hengyu Wang", "Honghe Ren", "Xiaohan Jia", "Xueyang Zhou", "Xiaoyu Cao", "Yongchao Chen", "Yuanning Feng", "Junhao Wu", "Cheng Zhang", "Sijia Chen", "Haoyu Xue", "Chengsong You", "Huan Wang", "Koutian Wu", "Peigan Gao", "Jiakun Wu", "Wenzhe Li", "Ergan Shang", "Qingyuan Zheng", "Jingjing Zhou", "Ruixuan Jia", "Yan Xu", "Hongrui Zhang", "Xiao-Han Ma", "Zhengxiang Cheng", "Yuexing Hao", "Liting Mai", "Xianglin Ji", "Wenjun Zhang", "Zhuofan Chen", "Yixiao Huang", "Chi Wang", "Wenyue Hua", "Yilun Hao", "Yuantao Zhai", "Ziyan Zhao", "Jingyan Xie"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.17271", "source_url": "https://arxiv.org/abs/2608.17271"}, "categories": ["AI Scientist", "Agents", "Agents & Tool Use", "Autonomous execution", "Reasoning", "Research planning", "Scientific Research & AI for Science", "Scientific discovery"], "description": {"en": "ASI-Bench evaluates AI systems on 60 project-level scientific research tasks across 11 domains, with four guidance levels B1-B4 measuring autonomous execution and innovation."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.17271", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.17271", "title": "ASI-Bench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2608.17271", "modality": null, "name": "ASI-Bench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-reviewed", "display_eligible": "unknown", "origin_record_id": "bm_asi-bench_faf0dd90", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "aa30422bc9b90ff39028bc3af5a541feb2a4ac5b066e312d1695da7aca6d7b70", "relation": "introduces", "review_state": "source-reviewed", "reviewed_at": "2026-08-19", "source_url": "https://arxiv.org/abs/2608.17271"}, "publication_dates": [{"basis": "paper_first_version", "date": "2026-08-18", "source_url": "https://arxiv.org/abs/2608.17271"}], "publisher": null, "released": "2026-08-18", "released_reference": {"basis": "first_public", "source_key": "claire-radar:2608.17271", "source_url": "https://arxiv.org/abs/2608.17271"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-17271", "source": "claire_radar", "source_benchmark_id": "2608.17271", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Expert Generated", "applicationDomains": ["Scientific Research & AI for Science"], "area": "Agents & Tool Use", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/apexin-ai/ASI-Bench", "githubScope": "benchmark_repo", "githubStars": 299, "hfDailySubmittedAt": null, "hfDataset": "Apexintelligence-AI/ASI-Bench-seed31415", "hfDatasetDownloads": 1780, "hfDatasetLikes": 20, "hfPaperUpvotes": 68, "hfPaperUrl": "https://huggingface.co/papers/2608.17271", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-30", "state": "stale"}, "hfDatasetLikes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-30", "state": "stale"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-30", "state": "stale"}}, "status": "fresh"}, "availability": {"evaluatorStatus": "available", "githubStatus": "available", "hfDatasetStatus": "available", "paperStatus": "available", "submissionStatus": "available"}, "capabilities": ["Scientific discovery", "Research planning", "Autonomous execution"], "confidence": "High", "construction": "Hybrid", "constructionDetail": "The maintainers distilled more than 1,300 candidate research ideas through five review rounds, over 1,100 review assignments, more than 2,000 task revisions, and over 1,500 sandbox runs.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "4ced4b176bd1572a50abbafd269d41d66f3dce2c7bc8ccc509a202924efd679e", "model": "deepseek-v4-flash"}, "curation": {"reviewedAt": "2026-08-19", "sources": ["https://github.com/apexin-ai/ASI-Bench", "https://arxiv.org/abs/2608.17271"], "state": "source-reviewed"}, "dataStatus": "primary-source-reviewed", "demo": false, "description": "ASI-Bench evaluates AI systems on 60 project-level scientific research tasks across 11 domains, with four guidance levels B1-B4 measuring autonomous execution and innovation.", "detail": {"adoption": {"independentOrganizations": [], "note": "No source-linked independent model-provider adoption has been recorded yet."}, "leaderboard": {"asOf": "2026-08-18", "assessment": "Benchmark-author assessment", "bestScore": 51.6, "bestSystem": "GPT-5.6 Sol (ultra) · Codex", "evaluatedConfigurations": 18, "meanScore": 26.62, "primaryMetric": "B3 Scientific Score", "saturationStatus": "Not saturated", "sourceUrl": "https://arxiv.org/abs/2608.17271"}, "leaderboardUrl": "https://asibench.apexin.ai/leaderboard", "modelCoverage": [{"models": ["GPT-5.5", "GPT-5.6 Sol"], "provider": "OpenAI"}, {"models": ["Claude Opus 4.8", "Claude Opus 5"], "provider": "Anthropic"}, {"models": ["Kimi K2.7", "Kimi K3"], "provider": "Moonshot AI"}, {"models": ["GLM-5.2", "GLM-5.3"], "provider": "Z.AI"}, {"models": ["DeepSeek V4 Flash", "DeepSeek V4 Pro"], "provider": "DeepSeek"}, {"models": ["MiniMax M3"], "provider": "MiniMax"}, {"models": ["MiMo V2.5 Pro"], "provider": "Xiaomi"}], "modelCoverageNote": "These systems were evaluated by the benchmark authors; their presence does not by itself mean the model provider adopted or endorsed ASI-Bench.", "protocol": {"aggregation": "Macro-average over tasks", "comparisonMetric": "B3 Scientific Score", "conditions": ["B1", "B2", "B3", "B4"], "primaryMetric": "Scientific Score", "runs": "Three independent runs unless marked otherwise", "tasks": 60, "tools": "Paper results reported without external tool access"}, "submissionUrl": "https://asibench.apexin.ai/submit", "taskBreakdown": ["Mathematics", "Physics", "Chemistry", "Biology", "Astronomy", "Materials science", "Earth science", "Medicine & biostatistics", "Computer science", "Robotics", "Electrical engineering"]}, "domainCuration": {"method": "primary-source-review-v1", "state": "reviewed"}, "evaluationMode": "score_submission", "evidence": {"reasonCodes": ["named benchmark release", "public evaluator", "submission path", "leaderboard"], "snippet": "ASI-Bench contains 60 project-level research tasks across 11 scientific domains and progressively reduces methodological guidance to test whether AI can independently select methods, conduct research, and produce verifiable results."}, "familyId": "bmf_3b83443d21ef", "firstSeenAt": "2026-08-19", "heat": null, "id": "bm_asi-bench_faf0dd90", "indexedAt": "2026-08-20T00:00:00Z", "industrySectors": ["Research & Development"], "links": {"code": "https://github.com/apexin-ai/ASI-Bench", "data": "https://huggingface.co/datasets/Apexintelligence-AI/ASI-Bench-seed31415", "hfPaper": "https://huggingface.co/papers/2608.17271", "paper": "https://arxiv.org/abs/2608.17271", "pdf": "https://arxiv.org/pdf/2608.17271", "project": "https://asibench.apexin.ai", "report": "https://arxiv.org/abs/2608.17271"}, "metrics": [{"name": "Tasks", "note": "project-level tasks", "value": "60"}, {"name": "Coverage", "note": "scientific domains", "value": "11"}, {"name": "Protocol", "note": "progressively less methodological guidance", "value": "B1–B4"}, {"name": "Evaluation", "note": "cross-review, AI-assisted audit, sandbox runs, scorer validation", "value": "Expert + execution"}], "motivation": "Evaluate whether AI agents can independently select methods, execute end-to-end research, and produce verifiable scientific results as human methodological guidance is progressively withdrawn.", "name": "ASI-Bench", "oneLine": "ASI-Bench evaluates AI systems on 60 project-level scientific research tasks across 11 domains, with four guidance levels B1-B4 measuring autonomous execution and innovation.", "paperTitle": "ASI-Bench: At the Dawn of Artificial Superintelligence", "primaryDomain": "Scientific Research & AI for Science", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.17271", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-08-20T00:00:00Z"}, "publications": [], "publishers": [{"name": "Apex Intelligence AI", "organizationType": "company-research-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/apexin-ai/ASI-Bench"}], "ranking": {"90d": {"confidence": "High", "coverage": 1.0, "datasetDownloadRank": 11, "datasetRankPopulation": 85, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.7613524284821, "value": 299}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": 0.770797543361211, "value": 1780}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.7481437953470013, "value": 68}}, "confidence": "High", "coverage": 1.0, "rank": 3, "score": 76}, "method": "attention-ranking-v13", "rank": 3, "score": 76}}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releaseDates": {"firstPublicAt": "2026-08-18", "paperV1At": "2026-08-18"}, "releasedAt": "2026-08-18", "source": {"authors": [], "categories": ["cs.AI"], "id": "2608.17271", "title": "ASI-Bench: At the Dawn of Artificial Superintelligence", "type": "arxiv", "url": "https://arxiv.org/abs/2608.17271"}, "sourceUpdatedAt": "2026-08-18T01:59:00Z", "topics": ["AI Scientist", "Agents", "Reasoning"], "venueAttempts": [], "whyItMatters": "It addresses the gap in evaluating AI's independent scientific exploration and execution, revealing current dependence on human guidance for end-to-end research."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
