{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2407.13168", "kind": "paper", "locator": "data/catalog/llm_stats_identity_overrides.yml#benchmarks.scicode.paper_url", "url": "https://arxiv.org/pdf/2407.13168"}, {"id": "gh:scicode-bench/SciCode", "kind": "repo", "locator": "data/catalog/llm_stats_identity_overrides.yml#benchmarks.scicode.repo_url", "url": "https://github.com/scicode-bench/SciCode"}], "categories": ["math", "physics", "reasoning", "biology", "chemistry", "code"], "description": {"en": "SciCode is a research coding benchmark curated by scientists that challenges language models to code solutions for scientific problems. It contains 338 subproblems decomposed from 80 challenging main problems across 16 natural science sub-fields including mathematics, physics, chemistry, biology, and materials science. Problems require knowledge recall, reasoning, and code synthesis skills."}, "documents": [{"document_type": "registry_page", "id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "title": "SciCode"}], "evidence_summary": {"document_count": 1, "model_count": 21, "model_count_basis": "source_model_id"}, "identity_override": {"candidate_matches": [], "evidence": [{"anchor_type": "paper", "claim": "The repository and paper define the same scientist-curated SciCode benchmark.", "url": "https://arxiv.org/pdf/2407.13168"}, {"anchor_type": "repo_readme", "claim": "The repository and paper define the same scientist-curated SciCode benchmark.", "url": "https://github.com/scicode-bench/SciCode"}], "note": "The repository and paper define the same scientist-curated SciCode benchmark.", "repository": {"full_name": "scicode-bench/SciCode", "kind": "benchmark_source", "resolution_status": "resolved", "subpath": null, "url": "https://github.com/scicode-bench/SciCode"}, "resolution_status": "resolved", "source": "reviewed_llm_stats_identity_override", "source_path": "data/catalog/llm_stats_identity_overrides.yml"}, "key": "llm-stats:scicode", "modality": "text", "name": "SciCode", "openness": {"access_gate": null, "basis": "reviewed_data_availability_not_established", "code_license": null, "code_license_note": null, "data_license": null, "data_license_note": null, "data_located": null, "evidence": ["https://github.com/scicode-bench/SciCode"], "link_status": null, "status": "unknown"}, "provenance": {"crawl_bundle": "llm_stats_2026-08-17", "crawled_at": "2026-08-17T18:49:35.489925+00:00", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500"}, "publisher": null, "released": "2024-07-18", "released_reference": {"basis": "paper_first_version", "note": "The reviewed source identity explicitly links this introducing SciCode paper. No score or adoption measurement is transferred from another source.", "source_key": "llm-stats:scicode", "source_url": "https://arxiv.org/abs/2407.13168"}, "repository": {"full_name": "scicode-bench/SciCode", "kind": "benchmark_source", "resolution_status": "resolved", "subpath": null, "url": "https://github.com/scicode-bench/SciCode"}, "schema_version": 1, "sizes": [], "slug": "llm-stats-scicode", "source": "llm_stats", "source_benchmark_id": "scicode"}, "schema_version": 1, "scores_by_source": {"llm_stats": {"rows": [{"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "seed-2.1-pro", "model_name": "Seed 2.1 Pro", "obs_id": "llm_stats:scicode:seed-2.1-pro", "organization": "ByteDance", "rank_in_source_response": 1, "raw_value": "0.598", "reported_by": "self_reported", "reported_date": "2026-06-24", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.598, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "gemini-3.1-pro-preview", "model_name": "Gemini 3.1 Pro", "obs_id": "llm_stats:scicode:gemini-3.1-pro-preview", "organization": "Google", "rank_in_source_response": 2, "raw_value": "0.59", "reported_by": "self_reported", "reported_date": "2026-02-19", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.59, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "seed-2.1-turbo", "model_name": "Seed 2.1 Turbo", "obs_id": "llm_stats:scicode:seed-2.1-turbo", "organization": "ByteDance", "rank_in_source_response": 3, "raw_value": "0.578", "reported_by": "self_reported", "reported_date": "2026-06-24", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.578, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "qwen3.7-max", "model_name": "Qwen3.7 Max", "obs_id": "llm_stats:scicode:qwen3.7-max", "organization": "Qwen", "rank_in_source_response": 4, "raw_value": "0.535", "reported_by": "self_reported", "reported_date": "2026-05-19", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.535, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "kimi-k2.6", "model_name": "Kimi K2.6", "obs_id": "llm_stats:scicode:kimi-k2.6", "organization": "Moonshot AI", "rank_in_source_response": 5, "raw_value": "0.522", "reported_by": "self_reported", "reported_date": "2026-04-20", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.522, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "qwen3.7-plus", "model_name": "Qwen3.7-Plus", "obs_id": "llm_stats:scicode:qwen3.7-plus", "organization": "Qwen", "rank_in_source_response": 6, "raw_value": "0.513", "reported_by": "self_reported", "reported_date": "2026-05-31", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.513, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "inkling-small", "model_name": "Inkling-Small", "obs_id": "llm_stats:scicode:inkling-small", "organization": "Thinking Machines Lab", "rank_in_source_response": 7, "raw_value": "0.487", "reported_by": "self_reported", "reported_date": "2026-07-30", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.487, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "kimi-k2.5", "model_name": "Kimi K2.5", "obs_id": "llm_stats:scicode:kimi-k2.5", "organization": "Moonshot AI", "rank_in_source_response": 8, "raw_value": "0.487", "reported_by": "self_reported", "reported_date": "2026-01-27", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.487, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "kimi-k2-thinking-0905", "model_name": "Kimi K2-Thinking-0905", "obs_id": "llm_stats:scicode:kimi-k2-thinking-0905", "organization": "Moonshot AI", "rank_in_source_response": 9, "raw_value": "0.448", "reported_by": "self_reported", "reported_date": "2025-09-05", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.448, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "nemotron-3-ultra-550b-a55b", "model_name": "Nemotron 3 Ultra (550B A55B)", "obs_id": "llm_stats:scicode:nemotron-3-ultra-550b-a55b", "organization": "NVIDIA", "rank_in_source_response": 10, "raw_value": "0.446", "reported_by": "self_reported", "reported_date": "2026-06-04", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.446, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "muse-glimmer-30b", "model_name": "Muse Glimmer-30B", "obs_id": "llm_stats:scicode:muse-glimmer-30b", "organization": "Meta", "rank_in_source_response": 11, "raw_value": "0.436", "reported_by": "self_reported", "reported_date": "2026-08-10", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.436, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "nemotron-3-super-120b-a12b", "model_name": "Nemotron 3 Super (120B A12B)", "obs_id": "llm_stats:scicode:nemotron-3-super-120b-a12b", "organization": "NVIDIA", "rank_in_source_response": 12, "raw_value": "0.4205", "reported_by": "self_reported", "reported_date": "2026-03-11", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.4205, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "glm-4.5", "model_name": "GLM-4.5", "obs_id": "llm_stats:scicode:glm-4.5", "organization": "Z.ai", "rank_in_source_response": 13, "raw_value": "0.417", "reported_by": "self_reported", "reported_date": "2025-07-28", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.417, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "minimax-m2.1", "model_name": "MiniMax M2.1", "obs_id": "llm_stats:scicode:minimax-m2.1", "organization": "MiniMax", "rank_in_source_response": 14, "raw_value": "0.39", "reported_by": "self_reported", "reported_date": "2025-12-23", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.39, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "north-mini-code-1.0", "model_name": "North Mini Code 1.0", "obs_id": "llm_stats:scicode:north-mini-code-1.0", "organization": "Cohere", "rank_in_source_response": 15, "raw_value": "0.382", "reported_by": "self_reported", "reported_date": "2026-06-09", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.382, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "command-a-plus-05-2026", "model_name": "Command A+", "obs_id": "llm_stats:scicode:command-a-plus-05-2026", "organization": "Cohere", "rank_in_source_response": 16, "raw_value": "0.38", "reported_by": "self_reported", "reported_date": "2026-05-20", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.38, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "mercury-2", "model_name": "Mercury 2", "obs_id": "llm_stats:scicode:mercury-2", "organization": "Inception", "rank_in_source_response": 17, "raw_value": "0.38", "reported_by": "self_reported", "reported_date": "2026-02-24", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.38, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "glm-4.5-air", "model_name": "GLM-4.5-Air", "obs_id": "llm_stats:scicode:glm-4.5-air", "organization": "Z.ai", "rank_in_source_response": 18, "raw_value": "0.373", "reported_by": "self_reported", "reported_date": "2025-07-28", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.373, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "minimax-m2", "model_name": "MiniMax M2", "obs_id": "llm_stats:scicode:minimax-m2", "organization": "MiniMax", "rank_in_source_response": 19, "raw_value": "0.36", "reported_by": "self_reported", "reported_date": "2025-10-27", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.36, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "nemotron-3-nano-30b-a3b", "model_name": "Nemotron 3 Nano (30B A3B)", "obs_id": "llm_stats:scicode:nemotron-3-nano-30b-a3b", "organization": "NVIDIA", "rank_in_source_response": 20, "raw_value": "0.333", "reported_by": "self_reported", "reported_date": "2025-12-15", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.333, "value_kind": "number"}, {"comparable_group": null, "crawled_at": "2026-08-17T18:49:35.489925+00:00", "date_precision": "model_announcement", "document_id": "llm_stats:https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "key": "llm-stats:scicode", "measured_by": null, "model_id": "nemotron-3.5-lightning-30b-a3b", "model_name": "Nemotron 3.5 Lightning (30B A3B)", "obs_id": "llm_stats:scicode:nemotron-3.5-lightning-30b-a3b", "organization": "NVIDIA", "rank_in_source_response": 21, "raw_value": "0.326", "reported_by": "self_reported", "reported_date": "2026-08-11", "series_id": "llm_stats:scicode:default", "source": "llm_stats", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500", "value": 0.326, "value_kind": "number"}], "series": {"bounds": {"basis": "aggregator_declared", "max": 1.0, "min": null}, "declared_max": 1.0, "direction": "higher_is_better", "direction_basis": "source_rank_descending", "display_scale": null, "first_score_record": {"date_precision": "model_announcement", "obs_id": "llm_stats:scicode:glm-4.5", "reported_at": "2025-07-28", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500"}, "first_score_report": null, "key": "llm-stats:scicode", "max_score_contradicted": false, "metric": null, "observation_count": 21, "observed_max": 0.598, "score_summary": {"display_max": 59.8, "display_multiplier": 100, "model_count": 21, "model_count_basis": "source_model_id", "numeric_count": 21, "raw_max": 0.598, "raw_min": 0.326, "source_reference": {"crawled_at": "2026-08-17T18:49:35.489925+00:00", "obs_id": "llm_stats:scicode:seed-2.1-pro", "reported_date": "2026-06-24", "source_url": "https://api.zeroeval.com/leaderboard/benchmarks/scicode?top_n=500"}, "unit": null}, "series_id": "llm_stats:scicode:default"}}}, "siblings": []}
