{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2606.06560", "kind": "paper", "url": "https://arxiv.org/abs/2606.06560"}, {"id": "gh:macpaw/macarena", "kind": "repo", "url": "https://github.com/MacPaw/MacArena"}], "authors": ["Victor Muryn", "Maksym Shamrai", "Sofiia Mazepa", "Yehor Khodysko"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2606.06560", "source_url": "https://arxiv.org/abs/2606.06560"}, "categories": ["Agents & Tool Use", "General AI", "cs.LG"], "description": {"en": "MacArena benchmarks computer-use agents on macOS with 421 manually verified tasks spanning 50 applications, running on Apple Virtualization framework on Apple Silicon."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2606.06560", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2606.06560", "title": "MacArena"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2606.06560", "modality": null, "name": "MacArena", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_macarena_a8690f15", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "8395c778e981dc21949f8f4b06d5cbd897453bba44af008ddd61c05d62c590cd", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2606.06560"}, "publisher": null, "released": "2026-06-04", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2606.06560", "source_url": "https://arxiv.org/abs/2606.06560"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2606-06560", "source": "claire_radar", "source_benchmark_id": "2606.06560", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Agents & Tool Use", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/MacPaw/MacArena", "githubScope": "benchmark_repo", "githubStars": 14, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 4, "hfPaperUrl": "https://huggingface.co/papers/2606.06560", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-25", "state": "stale"}}, "status": "fresh"}, "capabilities": [], "confidence": "High", "construction": "Transform Existing", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "c0a19fd0cd7a0e984a5aa7384b421bd1ef362b6ed51bb68f6bf9a022b9ece0dd", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "MacArena benchmarks computer-use agents on macOS with 421 manually verified tasks spanning 50 applications, running on Apple Virtualization framework on Apple Silicon.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["exact coined title identity tied to benchmark evidence", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "We introduce MacArena, a benchmark of 421 manually verified tasks spanning 50 applications that combines a curated port of OSWorld tasks, content sourced from macOSWorld, and 49 new macOS-native tasks, all running on Apple's native Virtualization framework on Apple Silicon."}, "familyId": "bmf_ff7f4b05b738", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "existing-code-link", "linkedUrl": "https://github.com/MacPaw/MacArena", "url": "https://github.com/MacPaw/MacArena"}, {"basis": "existing-metric-association", "linkedUrl": "https://github.com/MacPaw/MacArena", "url": "https://huggingface.co/papers/2606.06560"}], "scope": "benchmark_repo", "url": "https://github.com/MacPaw/MacArena", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_macarena_a8690f15", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": "https://github.com/MacPaw/MacArena", "data": null, "hfPaper": "https://huggingface.co/papers/2606.06560", "paper": "https://arxiv.org/abs/2606.06560", "pdf": "https://arxiv.org/pdf/2606.06560", "project": null, "report": "https://arxiv.org/abs/2606.06560"}, "metrics": [], "motivation": "Computer-use agents (CUAs) operate graphical user interfaces (GUIs) through vision and control primitives, and their capabilities have advanced rapidly, driven in part by standardized online evaluation benchmarks such as OSWorld, which serve both as evaluation tools and as training environments for reinforcement learning.", "name": "MacArena", "oneLine": "MacArena benchmarks computer-use agents on macOS with 421 manually verified tasks spanning 50 applications, running on Apple Virtualization framework on Apple Silicon.", "paperTitle": "MacArena: Benchmarking Computer Use Agents on an Online macOS Environment", "primaryDomain": "General AI", "publication": {"evidence": "Accepted to the Second Workshop on Agents in the Wild: Safety, Security, and Beyond (AIWILD) at ICML 2026", "evidenceLevel": "author-claim", "evidenceUrl": "https://arxiv.org/abs/2606.06560", "source": "arxiv-comments", "status": "acceptance_claimed", "venue": "Second Workshop on Agents in the Wild: Safety, Security, and Beyond (AIWILD) at ICML 2026", "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "MacPaw", "organizationType": "company-research-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/MacPaw/MacArena"}], "ranking": {}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-06-04", "source": {"authors": ["Victor Muryn", "Maksym Shamrai", "Sofiia Mazepa", "Yehor Khodysko"], "categories": ["cs.LG", "cs.AI", "cs.HC"], "id": "2606.06560", "title": "MacArena: Benchmarking Computer Use Agents on an Online macOS Environment", "type": "arxiv", "url": "https://arxiv.org/abs/2606.06560"}, "sourceUpdatedAt": "2026-06-04T14:01:32Z", "topics": ["cs.LG"], "venueAttempts": [{"decisionRaw": "Accepted to the Second Workshop on Agents in the Wild: Safety, Security, and Beyond (AIWILD) at ICML 2026", "evidence": [{"level": "author-claim", "observedAt": "2026-10-14T01:46:58.058199Z", "rawValue": "Accepted to the Second Workshop on Agents in the Wild: Safety, Security, and Beyond (AIWILD) at ICML 2026", "sourceType": "arxiv-comments", "sourceUrl": "https://arxiv.org/abs/2606.06560"}], "reviewStatus": "accepted", "venueName": "Second Workshop on Agents in the Wild: Safety, Security, and Beyond (AIWILD) at ICML 2026"}], "whyItMatters": "MacOS GUI challenges are underrepresented in current benchmarks. MacArena provides a harder and more diverse environment, revealing that performance on existing benchmarks may not generalize across platforms, aiding development of robust GUI agents."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
