{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2606.14397", "kind": "paper", "url": "https://arxiv.org/abs/2606.14397"}, {"id": "gh:gauntlet-benchmark/evaluation-harness", "kind": "repo", "url": "https://github.com/gauntlet-benchmark/evaluation-harness"}], "authors": ["Mykola Vysotskyi", "Runqi Lin", "Grzegorz Biziel", "Michal Zakrzewski", "Sebastian Montagna", "Damian Rynczak", "Shreyansh Padarha", "Kumail Alhamoud", "Zihao Fu", "William Lugoloobi", "Kai Rawal", "Hanna Yershova", "Taras Rumezhak", "Guohao Li", "Fazl Barez", "Baoyuan Wu", "Arkadiusz Drohomirecki", "Chris Russell", "Christopher Summerfield", "Adam Mahdi", "Volodymyr Karpiv", "Philip Torr", "Adel Bibi"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2606.14397", "source_url": "https://arxiv.org/abs/2606.14397"}, "categories": ["Agents", "General AI", "Geometric reasoning", "Reasoning", "Vision & 3D"], "description": {"en": "GauntletBench evaluates agent generalisation across five professional web applications with 100 vision-intensive tasks, probing temporal perception, graphical understanding, and 3D reasoning via automated objective scoring."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2606.14397", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2606.14397", "title": "GauntletBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2606.14397", "modality": null, "name": "GauntletBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_gauntletbench_be5725f6", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "9d640d452155422d519f93370c7b83c3de5b6fe5e14a312f9b20b45a6ec459f2", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2606.14397"}, "publisher": null, "released": "2026-06-12", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2606.14397", "source_url": "https://arxiv.org/abs/2606.14397"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2606-14397", "source": "claire_radar", "source_benchmark_id": "2606.14397", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Vision & 3D", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/gauntlet-benchmark/evaluation-harness", "githubScope": "benchmark_repo", "githubStars": 1, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 20, "hfPaperUrl": "https://huggingface.co/papers/2606.14397", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-10-01", "state": "stale"}}, "status": "fresh"}, "capabilities": ["Reasoning", "Geometric reasoning"], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "9993efbd5567a9dad8a4342b89529f7361b30868cc125412d1cfa395673fe155", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "GauntletBench evaluates agent generalisation across five professional web applications with 100 vision-intensive tasks, probing temporal perception, graphical understanding, and 3D reasoning via automated objective scoring.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["exact named benchmark artifact released in abstract", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "To this end, we introduce GauntletBench, a web-based benchmark for evaluating agent generalisation in challenging scenarios, focusing on three underexplored capabilities (temporal perception, graphical understanding, and 3D reasoning), across five less-covered professional applications (Video Editor, Workflow Builder, 3D Modeller, Flight Analyser, and Circuit Designer), each with 20 vision-intensive tasks (100 in total)."}, "familyId": "bmf_44f425325bfc", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "existing-code-link", "linkedUrl": "https://github.com/gauntlet-benchmark/evaluation-harness", "url": "https://github.com/gauntlet-benchmark/evaluation-harness"}, {"basis": "existing-metric-association", "linkedUrl": "https://github.com/gauntlet-benchmark/evaluation-harness", "url": "https://huggingface.co/papers/2606.14397"}], "scope": "project_repo", "url": "https://github.com/gauntlet-benchmark/evaluation-harness", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_gauntletbench_be5725f6", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": "https://github.com/gauntlet-benchmark/evaluation-harness", "data": null, "hfPaper": "https://huggingface.co/papers/2606.14397", "paper": "https://arxiv.org/abs/2606.14397", "pdf": "https://arxiv.org/pdf/2606.14397", "project": null, "report": "https://arxiv.org/abs/2606.14397"}, "metrics": [], "motivation": "As agentic systems continue to evolve and are widely deployed in real-world scenarios, there is a growing demand to faithfully evaluate their capabilities.", "name": "GauntletBench", "oneLine": "GauntletBench evaluates agent generalisation across five professional web applications with 100 vision-intensive tasks, probing temporal perception, graphical understanding, and 3D reasoning via automated objective scoring.", "paperTitle": "Running the Gauntlet: Re-evaluating the Capabilities of Agents Beyond Familiar Environments", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2606.14397", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-06-12", "source": {"authors": ["Mykola Vysotskyi", "Runqi Lin", "Grzegorz Biziel", "Michal Zakrzewski", "Sebastian Montagna", "Damian Rynczak", "Shreyansh Padarha", "Kumail Alhamoud", "Zihao Fu", "William Lugoloobi", "Kai Rawal", "Hanna Yershova", "Xander Davies", "Taras Rumezhak", "Guohao Li", "Fazl Barez", "Baoyuan Wu", "Arkadiusz Drohomirecki", "Yarin Gal", "Chris Russell", "Christopher Summerfield", "Adam Mahdi", "Volodymyr Karpiv", "Philip Torr", "Adel Bibi"], "categories": ["cs.LG"], "id": "2606.14397", "title": "Running the Gauntlet: Re-evaluating the Capabilities of Agents Beyond Familiar Environments", "type": "arxiv", "url": "https://arxiv.org/abs/2606.14397"}, "sourceUpdatedAt": "2026-06-25T10:49:26Z", "topics": ["Agents", "Reasoning"], "venueAttempts": [], "whyItMatters": "Existing agent benchmarks saturate and overlook harder capabilities; this benchmark reveals significant gaps in frontier agents, guiding development toward more robust real-world systems."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
