{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2608.17800", "kind": "paper", "url": "https://arxiv.org/abs/2608.17800"}, {"kind": "website", "url": "https://startupbench.github.io/"}], "authors": ["Liya Zhu", "Xin Ma", "Tao Liu", "Haodong Wang", "Ge Zhang", "Jingzhe Ding", "Qingshui Gu", "Yongjie Zhong", "Jinxiang Meng", "Yuan Gao", "Yunqiu Zhou", "Hao Zhu", "Jifeng He", "Yongzhi Liao", "Xinyi Zhang", "Chaoxin Li", "Yi Zhu", "Xi Lin", "Duju Zeng", "Xiang Gao", "Wen Zhang", "Yunyang Wang", "Duo Wang", "Huan Zhou", "Zuo Wang", "Jin Chen", "Kaiyuan Zhang", "Chuqian Yu", "Tianhao Yu", "Longxiang Liu", "Jianbo Xue", "Huimin Che", "Jiahao Wang", "Yujia Qin", "Jiaheng Liu", "Shen Yan", "Xiaolong Chang", "Wenhao Huang"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.17800", "source_url": "https://arxiv.org/abs/2608.17800"}, "categories": ["Agents", "Agents & Tool Use", "Complex instruction following", "Finance", "General AI", "Long-horizon task execution", "Professional artifact generation", "Scientific Research & AI for Science", "Tool use"], "description": {"en": "StartupBench is a benchmark for general-purpose agents on end-to-end workflows derived from market-validated AI startup products, with deliverable-oriented tasks and fine-grained rubrics."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.17800", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.17800", "title": "StartupBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2608.17800", "modality": null, "name": "StartupBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_startupbench_c9f7419c", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "630cf7df9ed88000cbfc14e4e06f25c4a0c1733adc8a8381adf41b973a7c5af7", "relation": "introduces", "review_state": "source-reviewed", "reviewed_at": "2026-08-20", "source_url": "https://arxiv.org/abs/2608.17800"}, "publisher": null, "released": "2026-08-18", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.17800", "source_url": "https://arxiv.org/abs/2608.17800"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-17800", "source": "claire_radar", "source_benchmark_id": "2608.17800", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI", "Finance", "Scientific Research & AI for Science"], "area": "Agents & Tool Use", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 12, "hfPaperUrl": "https://huggingface.co/papers/2608.17800", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-30", "state": "stale"}}, "status": "stale"}, "availability": {"evaluatorStatus": "described_not_released", "githubStatus": "not_found", "hfDatasetStatus": "not_found", "paperStatus": "available", "submissionStatus": "not_found"}, "capabilities": ["Long-horizon task execution", "Tool use", "Complex instruction following", "Professional artifact generation"], "confidence": "High", "construction": "Unknown", "constructionDetail": "StartupBench measures whether agents can complete realistic professional workflows and produce usable final work products across six broad disciplines.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "36c7ed0bbf540064241e50febdc82ed09641f45fd042dd87433ec946514c5077", "model": "deepseek-v4-flash"}, "curation": {"reviewedAt": "2026-08-20", "sources": ["https://arxiv.org/abs/2608.17800", "https://arxiv.org/html/2608.17800", "https://startupbench.github.io/"], "state": "source-reviewed"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "StartupBench is a benchmark for general-purpose agents on end-to-end workflows derived from market-validated AI startup products, with deliverable-oriented tasks and fine-grained rubrics.", "detail": {"leaderboardUrl": "https://startupbench.github.io/#leaderboard", "protocol": {"primaryMetric": "Importance-weighted rubric score (0–100); success rate at score ≥90", "tasks": "97 end-to-end workflow tasks", "version": "arXiv v1"}, "taskBreakdown": ["Medical & Healthcare", "Finance", "Legal", "Business & Management", "STEM & Computer Science", "Education & Humanities"]}, "domainCuration": {"method": "primary-source-review-v1", "state": "reviewed"}, "evaluationMode": "score_submission", "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "We introduce \\textbf{StartupBench}, an E2E agent benchmark grounded in market-validated AI startup products."}, "familyId": "bmf_064eca785918", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_startupbench_c9f7419c", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2608.17800", "paper": "https://arxiv.org/abs/2608.17800", "pdf": "https://arxiv.org/pdf/2608.17800", "project": "https://startupbench.github.io/", "report": "https://arxiv.org/abs/2608.17800"}, "metrics": [], "motivation": "Recent advances in Large Language Models(LLMs) and agents have substantially improved the ability of AI systems to execute complex tasks.", "name": "StartupBench", "oneLine": "Evaluates whether general-purpose agents can complete market-validated, end-to-end professional workflows and deliver usable work products.", "paperTitle": "StartupBench: Benchmarking General-Purpose Agents on Market-Validated End-to-End Workflows", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.17800", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {"90d": {"confidence": "Low", "coverage": 0.15, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": null, "value": null}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.4532127250650379, "value": 12}}, "confidence": "Low", "coverage": 0.15, "rank": null, "score": 49}, "method": "attention-ranking-v13", "rank": null, "score": 49}}, "readiness": "Inspectable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-08-18", "source": {"authors": ["Liya Zhu", "Xin Ma", "Tao Liu", "Haodong Wang", "Ge Zhang", "Jingzhe Ding", "Qingshui Gu", "Yongjie Zhong", "Jinxiang Meng", "Yuan Gao", "Yunqiu Zhou", "Hao Zhu", "Jifeng He", "Yongzhi Liao", "Xinyi Zhang", "Chaoxin Li", "Yi Zhu", "Xi Lin", "Duju Zeng", "Xiang Gao", "Wen Zhang", "Yunyang Wang", "Duo Wang", "Huan Zhou", "Zuo Wang", "Jin Chen", "Kaiyuan Zhang", "Chuqian Yu", "Tianhao Yu", "Longxiang Liu", "Jianbo Xue", "Huimin Che", "Jiahao Wang", "Yujia Qin", "Jiaheng Liu", "Shen Yan", "Xiaolong Chang", "Wenhao Huang"], "categories": ["cs.AI"], "id": "2608.17800", "title": "StartupBench: Benchmarking General-Purpose Agents on Market-Validated End-to-End Workflows", "type": "arxiv", "url": "https://arxiv.org/abs/2608.17800"}, "sourceUpdatedAt": "2026-08-18T14:01:32Z", "topics": ["Agents"], "venueAttempts": [], "whyItMatters": "Existing agent benchmarks are researcher-selected, leaving real-world task performance uncertain. StartupBench measures agents on tasks with demonstrated demand."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
