{"record": {"aliases": ["TruthfulQA"], "artifacts": [{"kind": "paper", "url": "https://arxiv.org/abs/2109.07958"}], "audits": [], "categories": ["factuality"], "description": {"en": "817 questions (arXiv:2607.28801, Table 1), the full benchmark. Measures how models mimic human falsehoods (arXiv:2109.07958). The study notes the difficulties of consistent human judgment (arXiv:2607.28801, section 2). The paper reports 18.7 / 21.0 over TruthfulQA MC1+MC2 combined (n=1634), not the standard single TruthfulQA metric, so no plain TruthfulQA rows are filed from it; the combined scores are filed under benchmark_id truthfulqa with instrument truthfulqa_mc1_mc2."}, "documents": [{"document_type": "technical_report", "id": "model_reports:benchmarks_are_not_monolithic_results", "model_name": "Llama-3.2-1B and SmolLM-1.7B-Instruct", "organization": "Aleph Alpha Research", "published": "2026-07-30", "retrieved_at": "2026-11-07", "source": "model_reports", "source_id": "benchmarks_are_not_monolithic_results", "source_url": "https://arxiv.org/abs/2607.28801", "title": "Llama-3.2-1B and SmolLM-1.7B-Instruct"}], "evidence_summary": {"document_count": 1, "model_count": 2, "model_count_basis": "source_model_id"}, "key": "model-reports:truthfulqa", "modality": null, "name": "TruthfulQA", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"registry": "data/model_cards.yml", "score_archive": "data/benchmark_scores.yml", "source_url": "https://arxiv.org/abs/2109.07958"}, "publisher": null, "released": "2021-09-08", "released_reference": {"basis": "benchmark_release", "source_key": "model-reports:truthfulqa", "source_url": "https://arxiv.org/abs/2109.07958"}, "schema_version": 1, "sizes": [], "slug": "truthfulqa", "source": "model_reports", "source_benchmark_id": "truthfulqa"}, "schema_version": 1, "scores_by_source": {"model_reports": {"rows": [{"benchmark_id": "truthfulqa", "comparable_group": "[\"model-reports:truthfulqa\",\"truthfulqa_mc1_mc2\",\"MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework\"]", "date_precision": "document_publication", "document_id": "model_reports:benchmarks_are_not_monolithic_results", "instrument": "truthfulqa_mc1_mc2", "key": "model-reports:truthfulqa", "measured_by": "Aleph Alpha Research", "measurement_kind": "reported_self_score", "model": "Llama-3.2-1B", "model_id": "[\"Meta\",\"Llama-3.2-1B\"]", "model_name": "Llama-3.2-1B", "obs_id": "curated\u0000truthfulqa\u0000benchmarks_are_not_monolithic_results\u0000truthfulqa_mc1_mc2\u0000MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework\u0000Llama-3.2-1B", "observation_id": "curated\u0000truthfulqa\u0000benchmarks_are_not_monolithic_results\u0000truthfulqa_mc1_mc2\u0000MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework\u0000Llama-3.2-1B", "organization": "Meta", "protocol": "MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework", "rank_in_source_response": null, "raw_value": "18.7", "read_from": "html_text", "reported_at": "2026-07-30", "reported_by": "third_party", "reported_date": "2026-07-30", "series_id": "model_reports:truthfulqa:default", "source": "model_reports", "source_document_type": "technical_report", "source_id": "benchmarks_are_not_monolithic_results", "source_title": "Llama-3.2-1B and SmolLM-1.7B-Instruct", "source_url": "https://arxiv.org/abs/2607.28801", "value": 18.7, "value_kind": "number"}, {"benchmark_id": "truthfulqa", "comparable_group": "[\"model-reports:truthfulqa\",\"truthfulqa_mc1_mc2\",\"MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework\"]", "date_precision": "document_publication", "document_id": "model_reports:benchmarks_are_not_monolithic_results", "instrument": "truthfulqa_mc1_mc2", "key": "model-reports:truthfulqa", "measured_by": "Aleph Alpha Research", "measurement_kind": "reported_self_score", "model": "SmolLM-1.7B-Instruct", "model_id": "[\"HuggingFaceTB\",\"SmolLM-1.7B-Instruct\"]", "model_name": "SmolLM-1.7B-Instruct", "obs_id": "curated\u0000truthfulqa\u0000benchmarks_are_not_monolithic_results\u0000truthfulqa_mc1_mc2\u0000MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework\u0000SmolLM-1.7B-Instruct", "observation_id": "curated\u0000truthfulqa\u0000benchmarks_are_not_monolithic_results\u0000truthfulqa_mc1_mc2\u0000MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework\u0000SmolLM-1.7B-Instruct", "organization": "HuggingFaceTB", "protocol": "MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework", "rank_in_source_response": null, "raw_value": "21.0", "read_from": "html_text", "reported_at": "2026-07-30", "reported_by": "third_party", "reported_date": "2026-07-30", "series_id": "model_reports:truthfulqa:default", "source": "model_reports", "source_document_type": "technical_report", "source_id": "benchmarks_are_not_monolithic_results", "source_title": "Llama-3.2-1B and SmolLM-1.7B-Instruct", "source_url": "https://arxiv.org/abs/2607.28801", "value": 21.0, "value_kind": "number"}], "series": {"declared_max": null, "direction": "higher_is_better", "direction_basis": "reported_metric", "display_scale": null, "first_score_record": {"date_precision": "document_publication", "obs_id": "curated\u0000truthfulqa\u0000benchmarks_are_not_monolithic_results\u0000truthfulqa_mc1_mc2\u0000MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework\u0000Llama-3.2-1B", "reported_at": "2026-07-30", "source_url": "https://arxiv.org/abs/2607.28801"}, "first_score_report": {"date_precision": "document_publication", "obs_id": "curated\u0000truthfulqa\u0000benchmarks_are_not_monolithic_results\u0000truthfulqa_mc1_mc2\u0000MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework\u0000Llama-3.2-1B", "reported_at": "2026-07-30", "source_url": "https://arxiv.org/abs/2607.28801"}, "key": "model-reports:truthfulqa", "max_score_contradicted": false, "metric": "accuracy", "observation_count": 2, "observed_max": 21.0, "score_summary": {"display_max": 21.0, "display_multiplier": 1, "model_count": 2, "model_count_basis": "source_model_id", "numeric_count": 2, "raw_max": 21.0, "raw_min": 18.7, "source_reference": {"obs_id": "curated\u0000truthfulqa\u0000benchmarks_are_not_monolithic_results\u0000truthfulqa_mc1_mc2\u0000MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework\u0000SmolLM-1.7B-Instruct", "observation_id": "curated\u0000truthfulqa\u0000benchmarks_are_not_monolithic_results\u0000truthfulqa_mc1_mc2\u0000MC1+MC2 combined (n=1634), loglikelihood, Aleph Alpha Eval-Framework\u0000SmolLM-1.7B-Instruct", "reported_at": "2026-07-30", "reported_date": "2026-07-30", "source_id": "benchmarks_are_not_monolithic_results", "source_url": "https://arxiv.org/abs/2607.28801"}, "unit": "percent"}, "series_id": "model_reports:truthfulqa:default", "unit": "percent"}}}, "siblings": []}