{"record": {"aliases": ["Beyond Correctness: Benchmarking and Aligning Response Behaviors in Hybrid-Thinking MLLMs"], "artifacts": [{"id": "arxiv:2608.12781", "kind": "paper", "url": "https://arxiv.org/abs/2608.12781"}], "authors": ["Xinming Wang", "Weinong Wang", "Hongming Yang", "Yansong Lin", "Zheng Ruan", "Shangpin Peng", "Qiming Peng", "Nan Qiao", "Fengyuan Lu", "Guoqing Ma", "Marito Li", "Songyang Zhang", "Saiyong Yang", "Han Hu", "Yonglong Tian", "Xu-Yao Zhang"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.12781", "source_url": "https://arxiv.org/abs/2608.12781"}, "categories": ["Factuality", "General AI", "Multimodal", "Reasoning"], "description": {"en": "PatternEval is a diagnostic benchmark with 2,415 multimodal prompts testing four response-pattern failures: chain-of-thought leakage, repetition, contradiction, and performative reasoning."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.12781", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.12781", "title": "PatternEval"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["Tencent (China)", "University of Electronic Science and Technology of China", "Hong Kong University of Science and Technology", "Chinese Academy of Sciences", "Institute of Automation"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2608.12781", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2608.12781", "source_url": "https://openalex.org/W7203465724"}, "key": "claire-radar:2608.12781", "modality": null, "name": "PatternEval", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "Medium", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "true", "origin_record_id": "bm_beyond-correctness-benchmarking-and-aligni_75c36d61", "origin_source": "arxiv", "recognition_confidence": "0.6", "record_sha256": "32df1b000b13627c8862e864a03050babbe69ef4f736acd9d8320d2e5ff14f31", "relation": "introduces", "review_model": "deepseek-v4-pro", "review_state": "ai-reviewed", "reviewed_at": "2026-08-26T06:10:57.123445Z", "source_url": "https://arxiv.org/abs/2608.12781"}, "publisher": null, "released": "2026-08-13", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.12781", "source_url": "https://arxiv.org/abs/2608.12781"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-12781", "source": "claire_radar", "source_benchmark_id": "2608.12781", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": ["Beyond Correctness: Benchmarking and Aligning Response Behaviors in Hybrid-Thinking MLLMs"], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Multimodal", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 35, "hfPaperUrl": "https://huggingface.co/papers/2608.12781", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-30", "state": "stale"}}, "status": "stale"}, "attentionForecast": {"confidence": "Medium", "horizon": "7d", "reason": "The benchmark addresses an emerging area of hybrid-thinking models and includes systematic failure analysis, likely to interest MLLM evaluation researchers.", "score": 50}, "capabilities": ["Reasoning", "Factuality"], "confidence": "Medium", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-26T06:10:57.123445Z", "inputHash": "0ffdaaa20734fa2298806ce7e50d0eda18ea3b3d2f71264baec1784996f335d9", "model": "deepseek-v4-pro"}, "curation": {"canonicalNameEvidence": "We introduce \\textbf{PatternEval}, a failure-enriched diagnostic benchmark comprising 2,415 multimodal prompts spanning visual perception and grounding, structured image understanding, and multimodal knowledge reasoning.", "canonicalNameSource": "abstract", "decisionReason": "The benchmark is explicitly named and introduced, and the paper presents a clear scoring protocol; however, the abstract does not mention public code or data release, though the benchmark is intended for reuse.", "model": "deepseek-v4-pro", "reviewedAt": "2026-08-26T06:10:57.123445Z", "state": "ai-reviewed"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "PatternEval is a diagnostic benchmark with 2,415 multimodal prompts testing four response-pattern failures: chain-of-thought leakage, repetition, contradiction, and performative reasoning.", "displayEligible": true, "domainCuration": {"method": "rules-v1", "state": "auto"}, "evaluationMode": "public_reusable", "evidence": {"reasonCodes": ["benchmark term in abstract", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "We introduce \\textbf{PatternEval}, a failure-enriched diagnostic benchmark comprising 2,415 multimodal prompts spanning visual perception and grounding, structured image understanding, and multimodal knowledge reasoning."}, "familyId": "bmf_fa1620ab6433", "firstSeenAt": "2026-08-19", "heat": null, "id": "bm_beyond-correctness-benchmarking-and-aligni_75c36d61", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": null, "data": null, "paper": "https://arxiv.org/abs/2608.12781", "pdf": "https://arxiv.org/pdf/2608.12781", "project": null, "report": "https://arxiv.org/abs/2608.12781"}, "metrics": [], "motivation": "Hybrid-thinking multimodal large language models (MLLMs) allow a single model to alternate between deliberative thinking and latency-efficient non-thinking inference.", "name": "PatternEval", "oneLine": "PatternEval is a diagnostic benchmark with 2,415 multimodal prompts testing four response-pattern failures: chain-of-thought leakage, repetition, contradiction, and performative reasoning.", "paperTitle": "Beyond Correctness: Benchmarking and Aligning Response Behaviors in Hybrid-Thinking MLLMs", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.12781", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-08-19T11:05:13.395391Z"}, "publications": [], "ranking": {"90d": {"confidence": "Low", "coverage": 0.15, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": null, "value": null}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.6331884794120033, "value": 35}}, "confidence": "Low", "coverage": 0.15, "rank": null, "score": 52}, "method": "attention-ranking-v13", "rank": null, "score": 52}}, "readiness": "Paper only", "recognitionConfidence": 0.6, "relation": "introduces", "releasedAt": "2026-08-13", "source": {"authors": ["Xinming Wang", "Weinong Wang", "Hongming Yang", "Yansong Lin", "Zheng Ruan", "Shangpin Peng", "Qiming Peng", "Nan Qiao", "Fengyuan Lu", "Guoqing Ma", "Marito Li", "Songyang Zhang", "Saiyong Yang", "Han Hu", "Yonglong Tian", "Xu-Yao Zhang"], "categories": ["cs.CV"], "id": "2608.12781", "title": "Beyond Correctness: Benchmarking and Aligning Response Behaviors in Hybrid-Thinking MLLMs", "type": "arxiv", "url": "https://arxiv.org/abs/2608.12781"}, "sourceUpdatedAt": "2026-08-17T18:08:51Z", "topics": ["Multimodal", "Reasoning"], "venueAttempts": [], "whyItMatters": "Evaluates response-pattern alignment between thinking and non-thinking modes in hybrid-thinking MLLMs, addressing user-facing quality beyond accuracy."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
