{"record": {"aliases": ["LatchBio scBench", "scBench (LatchBio)"], "artifacts": [{"kind": "repo", "url": "https://github.com/latchbio/scbench"}], "audits": [], "categories": ["biology"], "description": {"en": "195 single-cell RNA-seq analysis evaluations, but the full set is withheld to prevent training contamination and only six canonical examples are public, so scores cannot be independently reproduced. The published 95% intervals span about ±5 to 6.5 points, and the harness matters (GPT-5.5 scores 57.95 in mini-swe-agent and 57.78 in OpenAI Codex). The arXiv paper (2602.09063) reports an earlier task set, so its numbers are not comparable with these."}, "documents": [{"document_type": "benchmark_leaderboard", "id": "model_reports:latchbio_scbench_results_106726f", "model_name": null, "organization": "LatchBio", "published": "2026-11-16", "retrieved_at": "2026-11-18", "source": "model_reports", "source_id": "latchbio_scbench_results_106726f", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "title": "scBench results table (latchbio/scbench, commit 106726f)"}], "evidence_summary": {"document_count": 1, "model_count": 16, "model_count_basis": "source_model_id"}, "key": "model-reports:latchbio_scbench", "modality": null, "name": "LatchBio scBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"registry": "data/model_cards.yml", "score_archive": "data/benchmark_scores.yml", "source_url": "https://github.com/latchbio/scbench"}, "publisher": null, "released": null, "released_reference": null, "schema_version": 1, "sizes": [], "slug": "latchbio_scbench", "source": "model_reports", "source_benchmark_id": "latchbio_scbench"}, "schema_version": 1, "scores_by_source": {"model_reports": {"rows": [{"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; Claude Code harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Claude Opus 4.7", "model_id": "[\"Anthropic\",\"Claude Opus 4.7\"]", "model_name": "Claude Opus 4.7", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; Claude Code harness\u0000Claude Opus 4.7", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; Claude Code harness\u0000Claude Opus 4.7", "organization": "Anthropic", "protocol": "195 evaluations; Claude Code harness", "rank_in_source_response": null, "raw_value": "54.02", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 54.02, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; Claude Code harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Claude Opus 4.8", "model_id": "[\"Anthropic\",\"Claude Opus 4.8\"]", "model_name": "Claude Opus 4.8", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; Claude Code harness\u0000Claude Opus 4.8", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; Claude Code harness\u0000Claude Opus 4.8", "organization": "Anthropic", "protocol": "195 evaluations; Claude Code harness", "rank_in_source_response": null, "raw_value": "57.95", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 57.95, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; OpenAI Codex harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "GPT-5.5", "model_id": "[\"OpenAI\",\"GPT-5.5\"]", "model_name": "GPT-5.5", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; OpenAI Codex harness\u0000GPT-5.5", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; OpenAI Codex harness\u0000GPT-5.5", "organization": "OpenAI", "protocol": "195 evaluations; OpenAI Codex harness", "rank_in_source_response": null, "raw_value": "57.78", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 57.78, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Claude Opus 4.5", "model_id": "[\"Anthropic\",\"Claude Opus 4.5\"]", "model_name": "Claude Opus 4.5", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Opus 4.5", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Opus 4.5", "organization": "Anthropic", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "47.18", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 47.18, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Claude Opus 4.6", "model_id": "[\"Anthropic\",\"Claude Opus 4.6\"]", "model_name": "Claude Opus 4.6", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Opus 4.6", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Opus 4.6", "organization": "Anthropic", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "52.65", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 52.65, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Claude Opus 4.7", "model_id": "[\"Anthropic\",\"Claude Opus 4.7\"]", "model_name": "Claude Opus 4.7", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Opus 4.7", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Opus 4.7", "organization": "Anthropic", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "55.21", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 55.21, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Claude Opus 4.8", "model_id": "[\"Anthropic\",\"Claude Opus 4.8\"]", "model_name": "Claude Opus 4.8", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Opus 4.8", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Opus 4.8", "organization": "Anthropic", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "57.44", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 57.44, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Claude Sonnet 4.5", "model_id": "[\"Anthropic\",\"Claude Sonnet 4.5\"]", "model_name": "Claude Sonnet 4.5", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Sonnet 4.5", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Sonnet 4.5", "organization": "Anthropic", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "33.16", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 33.16, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Claude Sonnet 4.6", "model_id": "[\"Anthropic\",\"Claude Sonnet 4.6\"]", "model_name": "Claude Sonnet 4.6", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Sonnet 4.6", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Claude Sonnet 4.6", "organization": "Anthropic", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "50.26", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 50.26, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "GPT-5.1", "model_id": "[\"OpenAI\",\"GPT-5.1\"]", "model_name": "GPT-5.1", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000GPT-5.1", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000GPT-5.1", "organization": "OpenAI", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "38.8", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 38.8, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "GPT-5.2", "model_id": "[\"OpenAI\",\"GPT-5.2\"]", "model_name": "GPT-5.2", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000GPT-5.2", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000GPT-5.2", "organization": "OpenAI", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "52.31", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 52.31, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "GPT-5.4", "model_id": "[\"OpenAI\",\"GPT-5.4\"]", "model_name": "GPT-5.4", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000GPT-5.4", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000GPT-5.4", "organization": "OpenAI", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "57.44", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 57.44, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "GPT-5.5", "model_id": "[\"OpenAI\",\"GPT-5.5\"]", "model_name": "GPT-5.5", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000GPT-5.5", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000GPT-5.5", "organization": "OpenAI", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "57.95", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 57.95, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Gemini 2.5 Pro", "model_id": "[\"Google\",\"Gemini 2.5 Pro\"]", "model_name": "Gemini 2.5 Pro", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Gemini 2.5 Pro", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Gemini 2.5 Pro", "organization": "Google", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "23.59", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 23.59, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Gemini 3.1 Pro Preview", "model_id": "[\"Google\",\"Gemini 3.1 Pro Preview\"]", "model_name": "Gemini 3.1 Pro Preview", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Gemini 3.1 Pro Preview", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Gemini 3.1 Pro Preview", "organization": "Google", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "53.85", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 53.85, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Grok 4.1 Fast (reasoning)", "model_id": "[\"xAI\",\"Grok 4.1 Fast (reasoning)\"]", "model_name": "Grok 4.1 Fast (reasoning)", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Grok 4.1 Fast (reasoning)", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Grok 4.1 Fast (reasoning)", "organization": "xAI", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "30.26", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 30.26, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Grok 4.20 Beta 0309 (reasoning)", "model_id": "[\"xAI\",\"Grok 4.20 Beta 0309 (reasoning)\"]", "model_name": "Grok 4.20 Beta 0309 (reasoning)", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Grok 4.20 Beta 0309 (reasoning)", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Grok 4.20 Beta 0309 (reasoning)", "organization": "xAI", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "44.44", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 44.44, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; mini-swe-agent harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Grok 4.3", "model_id": "[\"xAI\",\"Grok 4.3\"]", "model_name": "Grok 4.3", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Grok 4.3", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; mini-swe-agent harness\u0000Grok 4.3", "organization": "xAI", "protocol": "195 evaluations; mini-swe-agent harness", "rank_in_source_response": null, "raw_value": "44.27", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 44.27, "value_kind": "number"}, {"benchmark_id": "latchbio_scbench", "comparable_group": "[\"model-reports:latchbio_scbench\",\"latchbio_scbench_195_evals\",\"195 evaluations; pi harness\"]", "date_precision": "document_publication", "document_id": "model_reports:latchbio_scbench_results_106726f", "instrument": "latchbio_scbench_195_evals", "key": "model-reports:latchbio_scbench", "measured_by": "LatchBio", "measurement_kind": "benchmark_publisher_run", "model": "Gemini 3.5 Flash", "model_id": "[\"Google\",\"Gemini 3.5 Flash\"]", "model_name": "Gemini 3.5 Flash", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; pi harness\u0000Gemini 3.5 Flash", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; pi harness\u0000Gemini 3.5 Flash", "organization": "Google", "protocol": "195 evaluations; pi harness", "rank_in_source_response": null, "raw_value": "56.92", "read_from": "html_text", "reported_at": "2026-11-16", "reported_by": "third_party", "reported_date": "2026-11-16", "series_id": "model_reports:latchbio_scbench:default", "source": "model_reports", "source_document_type": "benchmark_leaderboard", "source_id": "latchbio_scbench_results_106726f", "source_title": "scBench results table (latchbio/scbench, commit 106726f)", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv", "value": 56.92, "value_kind": "number"}], "series": {"declared_max": null, "direction": "higher_is_better", "direction_basis": "reported_metric", "display_scale": null, "first_score_record": {"date_precision": "document_publication", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; Claude Code harness\u0000Claude Opus 4.7", "reported_at": "2026-11-16", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv"}, "first_score_report": {"date_precision": "document_publication", "obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; Claude Code harness\u0000Claude Opus 4.7", "reported_at": "2026-11-16", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv"}, "key": "model-reports:latchbio_scbench", "max_score_contradicted": false, "metric": "accuracy", "observation_count": 19, "observed_max": 57.95, "score_summary": {"display_max": 57.95, "display_multiplier": 1, "model_count": 16, "model_count_basis": "source_model_id", "numeric_count": 19, "raw_max": 57.95, "raw_min": 23.59, "source_reference": {"obs_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; Claude Code harness\u0000Claude Opus 4.8", "observation_id": "curated\u0000latchbio_scbench\u0000latchbio_scbench_results_106726f\u0000latchbio_scbench_195_evals\u0000195 evaluations; Claude Code harness\u0000Claude Opus 4.8", "reported_at": "2026-11-16", "reported_date": "2026-11-16", "source_id": "latchbio_scbench_results_106726f", "source_url": "https://github.com/latchbio/scbench/blob/106726f432e0221a3fdbf9ef8e39b21f638f9b60/results/model_results.csv"}, "unit": "percent"}, "series_id": "model_reports:latchbio_scbench:default", "unit": "percent"}}}, "siblings": []}
