{"record": {"aliases": [], "artifacts": [{"id": "gh:xlang-ai/OSWorld", "kind": "repo", "locator": "data/catalog/llm_stats_identity_overrides.yml#benchmarks.osworld.repo_url", "url": "https://github.com/xlang-ai/OSWorld"}], "categories": ["multimodal", "general", "agents", "vision"], "description": {"en": "OSWorld: The first-of-its-kind scalable, real computer environment for multimodal agents, supporting task setup, execution-based evaluation, and interactive learning across Ubuntu, Windows, and macOS with 369 computer tasks involving real web and desktop applications, OS file I/O, and multi-application workflows"}, "documents": [{"document_type": "registry_page", "id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "title": "OSWorld"}], "evidence_summary": {"document_count": 1, "model_count": 20, "model_count_basis": "source_model_id"}, "identity_override": {"candidate_matches": [], "evidence": [{"anchor_type": "repo_readme", "claim": "The official repository and project site publish OSWorld tasks and evaluation.", "url": "https://github.com/xlang-ai/OSWorld"}, {"anchor_type": "project_site", "claim": "The official repository and project site publish OSWorld tasks and evaluation.", "url": "https://osworld-v1.xlang.ai/"}], "note": "The official repository and project site publish OSWorld tasks and evaluation.", "repository": {"full_name": "xlang-ai/OSWorld", "kind": "benchmark_source", "resolution_status": "resolved", "subpath": null, "url": "https://github.com/xlang-ai/OSWorld"}, "resolution_status": "resolved", "source": "reviewed_llm_stats_identity_override", "source_path": "data/catalog/llm_stats_identity_overrides.yml"}, "key": "llm-stats:osworld", "modality": "multimodal", "name": "OSWorld", "openness": {"access_gate": null, "basis": "reviewed_data_availability_not_established", "code_license": null, "code_license_note": null, "data_license": null, "data_license_note": null, "data_located": null, "evidence": ["https://github.com/xlang-ai/OSWorld"], "link_status": null, "status": "unknown"}, "provenance": {"crawl_bundle": "llm_stats_2026-08-17", "crawled_at": "2026-08-17T18:49:35.489925+00:00", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500"}, "publisher": null, "released": "2024-04-11", "released_reference": {"basis": "paper_first_version", "note": "First version of the introducing paper: OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments. The source-record description identifies this benchmark or its original partition; no dates or scores are joined by name at runtime.", "source_key": "llm-stats:osworld", "source_url": "https://arxiv.org/abs/2404.07972"}, "repository": {"full_name": "xlang-ai/OSWorld", "kind": "benchmark_source", "resolution_status": "resolved", "subpath": null, "url": "https://github.com/xlang-ai/OSWorld"}, "schema_version": 1, "sizes": [], "slug": "llm-stats-osworld", "source": "llm_stats", "source_benchmark_id": "osworld"}, "schema_version": 1, "scores_by_source": {"llm_stats": {"rows": [{"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "seed-2.1-pro", "model_name": "Seed 2.1 Pro", "obs_id": "llm_stats:osworld:seed-2.1-pro", "organization": "ByteDance", "rank_in_source_response": 1, "raw_value": "0.788", "reported_by": "self_reported", "reported_date": "2026-06-24", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.788, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "seed-2.1-turbo", "model_name": "Seed 2.1 Turbo", "obs_id": "llm_stats:osworld:seed-2.1-turbo", "organization": "ByteDance", "rank_in_source_response": 2, "raw_value": "0.764", "reported_by": "self_reported", "reported_date": "2026-06-24", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.764, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "claude-opus-4-6", "model_name": "Claude Opus 4.6", "obs_id": "llm_stats:osworld:claude-opus-4-6", "organization": "Anthropic", "rank_in_source_response": 3, "raw_value": "0.727", "reported_by": "self_reported", "reported_date": "2026-02-05", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.727, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "claude-sonnet-4-6", "model_name": "Claude Sonnet 4.6", "obs_id": "llm_stats:osworld:claude-sonnet-4-6", "organization": "Anthropic", "rank_in_source_response": 4, "raw_value": "0.725", "reported_by": "self_reported", "reported_date": "2026-02-17", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.725, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen3-vl-235b-a22b-instruct", "model_name": "Qwen3 VL 235B A22B Instruct", "obs_id": "llm_stats:osworld:qwen3-vl-235b-a22b-instruct", "organization": "Qwen", "rank_in_source_response": 5, "raw_value": "0.667", "reported_by": "self_reported", "reported_date": "2025-09-22", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.667, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "claude-opus-4-5-20251101", "model_name": "Claude Opus 4.5", "obs_id": "llm_stats:osworld:claude-opus-4-5-20251101", "organization": "Anthropic", "rank_in_source_response": 6, "raw_value": "0.663", "reported_by": "self_reported", "reported_date": "2025-11-24", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.663, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "glm-5v-turbo", "model_name": "GLM-5V-Turbo", "obs_id": "llm_stats:osworld:glm-5v-turbo", "organization": "Z.ai", "rank_in_source_response": 7, "raw_value": "0.623", "reported_by": "self_reported", "reported_date": "2026-04-02", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.623, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "claude-sonnet-4-5-20250929", "model_name": "Claude Sonnet 4.5", "obs_id": "llm_stats:osworld:claude-sonnet-4-5-20250929", "organization": "Anthropic", "rank_in_source_response": 8, "raw_value": "0.614", "reported_by": "self_reported", "reported_date": "2025-09-29", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.614, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "claude-haiku-4-5-20251001", "model_name": "Claude Haiku 4.5", "obs_id": "llm_stats:osworld:claude-haiku-4-5-20251001", "organization": "Anthropic", "rank_in_source_response": 9, "raw_value": "0.507", "reported_by": "self_reported", "reported_date": "2025-10-15", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.507, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen3-vl-32b-thinking", "model_name": "Qwen3 VL 32B Thinking", "obs_id": "llm_stats:osworld:qwen3-vl-32b-thinking", "organization": "Qwen", "rank_in_source_response": 10, "raw_value": "0.41", "reported_by": "self_reported", "reported_date": "2025-09-22", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.41, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen3-vl-235b-a22b-thinking", "model_name": "Qwen3 VL 235B A22B Thinking", "obs_id": "llm_stats:osworld:qwen3-vl-235b-a22b-thinking", "organization": "Qwen", "rank_in_source_response": 11, "raw_value": "0.381", "reported_by": "self_reported", "reported_date": "2025-09-22", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.381, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen3-vl-8b-thinking", "model_name": "Qwen3 VL 8B Thinking", "obs_id": "llm_stats:osworld:qwen3-vl-8b-thinking", "organization": "Qwen", "rank_in_source_response": 12, "raw_value": "0.339", "reported_by": "self_reported", "reported_date": "2025-09-22", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.339, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen3-vl-8b-instruct", "model_name": "Qwen3 VL 8B Instruct", "obs_id": "llm_stats:osworld:qwen3-vl-8b-instruct", "organization": "Qwen", "rank_in_source_response": 13, "raw_value": "0.339", "reported_by": "self_reported", "reported_date": "2025-09-22", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.339, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen3-vl-32b-instruct", "model_name": "Qwen3 VL 32B Instruct", "obs_id": "llm_stats:osworld:qwen3-vl-32b-instruct", "organization": "Qwen", "rank_in_source_response": 14, "raw_value": "0.326", "reported_by": "self_reported", "reported_date": "2025-09-22", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.326, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen3-vl-4b-thinking", "model_name": "Qwen3 VL 4B Thinking", "obs_id": "llm_stats:osworld:qwen3-vl-4b-thinking", "organization": "Qwen", "rank_in_source_response": 15, "raw_value": "0.314", "reported_by": "self_reported", "reported_date": "2025-09-22", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.314, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen3-vl-30b-a3b-thinking", "model_name": "Qwen3 VL 30B A3B Thinking", "obs_id": "llm_stats:osworld:qwen3-vl-30b-a3b-thinking", "organization": "Qwen", "rank_in_source_response": 16, "raw_value": "0.306", "reported_by": "self_reported", "reported_date": "2025-09-22", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.306, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen3-vl-30b-a3b-instruct", "model_name": "Qwen3 VL 30B A3B Instruct", "obs_id": "llm_stats:osworld:qwen3-vl-30b-a3b-instruct", "organization": "Qwen", "rank_in_source_response": 17, "raw_value": "0.303", "reported_by": "self_reported", "reported_date": "2025-09-22", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.303, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen3-vl-4b-instruct", "model_name": "Qwen3 VL 4B Instruct", "obs_id": "llm_stats:osworld:qwen3-vl-4b-instruct", "organization": "Qwen", "rank_in_source_response": 18, "raw_value": "0.262", "reported_by": "self_reported", "reported_date": "2025-09-22", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.262, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen2.5-vl-72b", "model_name": "Qwen2.5 VL 72B Instruct", "obs_id": "llm_stats:osworld:qwen2.5-vl-72b", "organization": "Qwen", "rank_in_source_response": 19, "raw_value": "0.0883", "reported_by": "self_reported", "reported_date": "2025-01-26", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.0883, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "key": "llm-stats:osworld", "measured_by": null, "model_id": "qwen2.5-vl-32b", "model_name": "Qwen2.5 VL 32B Instruct", "obs_id": "llm_stats:osworld:qwen2.5-vl-32b", "organization": "Qwen", "rank_in_source_response": 20, "raw_value": "0.0592", "reported_by": "self_reported", "reported_date": "2025-02-28", "series_id": "llm_stats:osworld:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500", "value": 0.0592, "value_kind": "number"}], "series": {"bounds": {"basis": "aggregator_declared", "max": 1.0, "min": null}, "declared_max": 1.0, "direction": "higher_is_better", "direction_basis": "source_rank_descending", "display_scale": null, "first_score_record": {"date_precision": "model_announcement", "obs_id": "llm_stats:osworld:qwen2.5-vl-72b", "reported_at": "2025-01-26", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500"}, "first_score_report": null, "key": "llm-stats:osworld", "max_score_contradicted": false, "metric": null, "observation_count": 20, "observed_max": 0.788, "score_summary": {"display_max": 78.8, "display_multiplier": 100, "model_count": 20, "model_count_basis": "source_model_id", "numeric_count": 20, "raw_max": 0.788, "raw_min": 0.0592, "source_reference": {"crawled_at": "2026-08-17T18:49:35.489925+00:00", "obs_id": "llm_stats:osworld:seed-2.1-pro", "reported_date": "2026-06-24", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/osworld?top_n=500"}, "unit": null}, "series_id": "llm_stats:osworld:default"}}}, "siblings": [{"key": "opencompass:1625", "name": "ubuntu_osworld", "relation": "variant:of", "slug": "opencompass-1625-ubuntu-osworld", "source": "opencompass_hub"}]}
