{"record": {"aliases": ["WinoGrande", "Winogrande"], "artifacts": [{"kind": "paper", "url": "https://arxiv.org/abs/1907.10641"}], "audits": [], "categories": ["reasoning"], "description": {"en": "1,270 problems is the sample count the study audited (arXiv:2607.28801, Table 1), not the full dataset of 44k problems (arXiv:1907.10641). An adversarial Winograd Schema challenge (arXiv:1907.10641). The study notes analyses reveal persistent gender and cultural biases (arXiv:2607.28801, section 2). The annotating paper also reports full-split accuracies for Llama-3.2-1B and SmolLM-1.7B-Instruct (arXiv:2607.28801, Table 4 / App. Table 9); those rows are filed in benchmark_scores.yml."}, "documents": [{"document_type": "technical_report", "id": "model_reports:benchmarks_are_not_monolithic_results", "model_name": "Llama-3.2-1B and SmolLM-1.7B-Instruct", "organization": "Aleph Alpha Research", "published": "2026-07-30", "retrieved_at": "2026-11-07", "source": "model_reports", "source_id": "benchmarks_are_not_monolithic_results", "source_url": "https://arxiv.org/abs/2607.28801", "title": "Llama-3.2-1B and SmolLM-1.7B-Instruct"}], "evidence_summary": {"document_count": 1, "model_count": 2, "model_count_basis": "source_model_id"}, "key": "model-reports:winogrande", "modality": null, "name": "WinoGrande", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"registry": "data/model_cards.yml", "score_archive": "data/benchmark_scores.yml", "source_url": "https://arxiv.org/abs/1907.10641"}, "publisher": null, "released": "2019-07-24", "released_reference": {"basis": "benchmark_release", "source_key": "model-reports:winogrande", "source_url": "https://arxiv.org/abs/1907.10641"}, "schema_version": 1, "sizes": [], "slug": "winogrande", "source": "model_reports", "source_benchmark_id": "winogrande"}, "schema_version": 1, "scores_by_source": {"model_reports": {"rows": [{"benchmark_id": "winogrande", "comparable_group": "[\"model-reports:winogrande\",\"winogrande\",\"winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework\"]", "date_precision": "document_publication", "document_id": "model_reports:benchmarks_are_not_monolithic_results", "instrument": "winogrande", "key": "model-reports:winogrande", "measured_by": "Aleph Alpha Research", "measurement_kind": "reported_self_score", "model": "Llama-3.2-1B", "model_id": "[\"Meta\",\"Llama-3.2-1B\"]", "model_name": "Llama-3.2-1B", "obs_id": "curated\u0000winogrande\u0000benchmarks_are_not_monolithic_results\u0000winogrande\u0000winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework\u0000Llama-3.2-1B", "observation_id": "curated\u0000winogrande\u0000benchmarks_are_not_monolithic_results\u0000winogrande\u0000winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework\u0000Llama-3.2-1B", "organization": "Meta", "protocol": "winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework", "rank_in_source_response": null, "raw_value": "57.9", "read_from": "html_text", "reported_at": "2026-07-30", "reported_by": "third_party", "reported_date": "2026-07-30", "series_id": "model_reports:winogrande:default", "source": "model_reports", "source_document_type": "technical_report", "source_id": "benchmarks_are_not_monolithic_results", "source_title": "Llama-3.2-1B and SmolLM-1.7B-Instruct", "source_url": "https://arxiv.org/abs/2607.28801", "value": 57.9, "value_kind": "number"}, {"benchmark_id": "winogrande", "comparable_group": "[\"model-reports:winogrande\",\"winogrande\",\"winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework\"]", "date_precision": "document_publication", "document_id": "model_reports:benchmarks_are_not_monolithic_results", "instrument": "winogrande", "key": "model-reports:winogrande", "measured_by": "Aleph Alpha Research", "measurement_kind": "reported_self_score", "model": "SmolLM-1.7B-Instruct", "model_id": "[\"HuggingFaceTB\",\"SmolLM-1.7B-Instruct\"]", "model_name": "SmolLM-1.7B-Instruct", "obs_id": "curated\u0000winogrande\u0000benchmarks_are_not_monolithic_results\u0000winogrande\u0000winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework\u0000SmolLM-1.7B-Instruct", "observation_id": "curated\u0000winogrande\u0000benchmarks_are_not_monolithic_results\u0000winogrande\u0000winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework\u0000SmolLM-1.7B-Instruct", "organization": "HuggingFaceTB", "protocol": "winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework", "rank_in_source_response": null, "raw_value": "55.0", "read_from": "html_text", "reported_at": "2026-07-30", "reported_by": "third_party", "reported_date": "2026-07-30", "series_id": "model_reports:winogrande:default", "source": "model_reports", "source_document_type": "technical_report", "source_id": "benchmarks_are_not_monolithic_results", "source_title": "Llama-3.2-1B and SmolLM-1.7B-Instruct", "source_url": "https://arxiv.org/abs/2607.28801", "value": 55.0, "value_kind": "number"}], "series": {"declared_max": null, "direction": "higher_is_better", "direction_basis": "reported_metric", "display_scale": null, "first_score_record": {"date_precision": "document_publication", "obs_id": "curated\u0000winogrande\u0000benchmarks_are_not_monolithic_results\u0000winogrande\u0000winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework\u0000Llama-3.2-1B", "reported_at": "2026-07-30", "source_url": "https://arxiv.org/abs/2607.28801"}, "first_score_report": {"date_precision": "document_publication", "obs_id": "curated\u0000winogrande\u0000benchmarks_are_not_monolithic_results\u0000winogrande\u0000winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework\u0000Llama-3.2-1B", "reported_at": "2026-07-30", "source_url": "https://arxiv.org/abs/2607.28801"}, "key": "model-reports:winogrande", "max_score_contradicted": false, "metric": "accuracy", "observation_count": 2, "observed_max": 57.9, "score_summary": {"display_max": 57.9, "display_multiplier": 1, "model_count": 2, "model_count_basis": "source_model_id", "numeric_count": 2, "raw_max": 57.9, "raw_min": 55.0, "source_reference": {"obs_id": "curated\u0000winogrande\u0000benchmarks_are_not_monolithic_results\u0000winogrande\u0000winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework\u0000Llama-3.2-1B", "observation_id": "curated\u0000winogrande\u0000benchmarks_are_not_monolithic_results\u0000winogrande\u0000winogrande_xl test split, loglikelihood, Aleph Alpha Eval-Framework\u0000Llama-3.2-1B", "reported_at": "2026-07-30", "reported_date": "2026-07-30", "source_id": "benchmarks_are_not_monolithic_results", "source_url": "https://arxiv.org/abs/2607.28801"}, "unit": "percent"}, "series_id": "model_reports:winogrande:default", "unit": "percent"}}}, "siblings": []}