{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2607.20526", "kind": "paper", "url": "https://arxiv.org/abs/2607.20526"}], "authors": ["Matthew ffrench-Constant", "Daniel Yang", "Xinmeng Huang", "Sanyam Kapoor"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2607.20526", "source_url": "https://arxiv.org/abs/2607.20526"}, "categories": ["General AI", "Language & Knowledge", "cs.AI"], "description": {"en": "A calibration benchmark evaluating verbalized confidence estimates in frontier LLMs using Brier scores across 200 multiple-choice questions in four categories. Scores are elicited via prompting without logits, applicable to closed and open models."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2607.20526", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2607.20526", "title": "ConfidenceBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["ETH Zurich", "New York University", "University of Pennsylvania"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2607.20526", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2607.20526", "source_url": "https://openalex.org/W7170385212"}, "key": "claire-radar:2607.20526", "modality": null, "name": "ConfidenceBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_confidencebench_21fa8523", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "2e842d65c9dae21ec47f4c3f36f32f3ba7810cd0f938c0b39c07e09e13be886e", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2607.20526"}, "publisher": null, "released": "2026-07-10", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2607.20526", "source_url": "https://arxiv.org/abs/2607.20526"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2607-20526", "source": "claire_radar", "source_benchmark_id": "2607.20526", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2607.20526", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "unavailable"}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "f1f68150dee752cb5c5721bba768cead5af118428559027ee8c9d7762480569b", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "A calibration benchmark evaluating verbalized confidence estimates in frontier LLMs using Brier scores across 200 multiple-choice questions in four categories. Scores are elicited via prompting without logits, applicable to closed and open models.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "We present ConfidenceBench, a calibration benchmark that evaluates verbalized confidence estimates in 15 frontier LLMs using the Brier score, a proper scoring rule that incentivises truthful probability reporting."}, "familyId": "bmf_b9ca48dc2fb1", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_confidencebench_21fa8523", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2607.20526", "paper": "https://arxiv.org/abs/2607.20526", "pdf": "https://arxiv.org/pdf/2607.20526", "project": null, "report": "https://arxiv.org/abs/2607.20526"}, "metrics": [], "motivation": "Large language models (LLMs) are increasingly deployed in settings where fluent but incorrect answers can be costly.", "name": "ConfidenceBench", "oneLine": "A calibration benchmark evaluating verbalized confidence estimates in frontier LLMs using Brier scores across 200 multiple-choice questions in four categories. Scores are elicited via prompting without logits, applicable to closed and open models.", "paperTitle": "ConfidenceBench: Evaluating Confidence Calibration in Large Language Models", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2607.20526", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {"90d": {"confidence": "Low", "coverage": 0.0, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": null, "value": null}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "rank": null, "score": null}, "method": "attention-ranking-v13", "rank": null, "score": null}}, "readiness": "Paper only", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-07-10", "source": {"authors": ["Matthew ffrench-Constant", "Daniel Yang", "Xinmeng Huang", "Sanyam Kapoor"], "categories": ["cs.AI", "cs.LG", "stat.ML"], "id": "2607.20526", "title": "ConfidenceBench: Evaluating Confidence Calibration in Large Language Models", "type": "arxiv", "url": "https://arxiv.org/abs/2607.20526"}, "sourceUpdatedAt": "2026-07-10T01:56:30Z", "topics": ["cs.AI"], "venueAttempts": [], "whyItMatters": "The benchmark addresses the need to assess model calibration separately from accuracy, which is critical for trustworthy deployment. The private nature of the questions and lack of public artifacts prevent other teams from running or inspecting the benchmark, limiting its standalone utility."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
