{"record": {"aliases": ["OmniJudge or OmniBias? Diagnosing Multimodal Judges through Balanced, Decoupled Lenses"], "artifacts": [{"id": "arxiv:2608.24160", "kind": "paper", "url": "https://arxiv.org/abs/2608.24160"}], "authors": ["Guangzheng Hu", "Ziyue Jiang", "Weixu Qiao", "Lixin Zhang", "Jianye Kang", "Yuru Wu", "Rong Bao", "Niantong Li", "Wei Wang", "Ziyi Cheng", "Xinfa Zhu", "HangRui Hu", "Ting He", "Bing Zhao", "Lin Qu", "Hu Wei", "Jin Xu"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.24160", "source_url": "https://arxiv.org/abs/2608.24160"}, "categories": ["General AI", "Multimodal"], "description": {"en": "D3-Omni diagnoses multimodal judge models across T2I, T2V, and TTS tasks with 53 balanced orthogonal binary dimensions and 10,671 samples that isolate specific capability failures."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.24160", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.24160", "title": "D3-Omni"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["The University of Melbourne", "Zhejiang University", "Alibaba DAMO Academy (China)"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2608.24160", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2608.24160", "source_url": "https://openalex.org/W7204266032"}, "key": "claire-radar:2608.24160", "modality": null, "name": "D3-Omni", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "Medium", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "false", "origin_record_id": "bm_omnijudge-or-omnibias-diagnosing-multimoda_74cb9dd9", "origin_source": "arxiv", "recognition_confidence": "0.6", "record_sha256": "5bce0ca14887ad790a6c010c6ab93f765d019376bb3585be2ce23245a93514c0", "relation": "introduces", "review_model": "deepseek-v4-pro", "review_state": "ai-name-audit-deferred", "reviewed_at": "2026-08-27T04:18:56.318436Z", "source_url": "https://arxiv.org/abs/2608.24160"}, "publisher": null, "released": "2026-08-25", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.24160", "source_url": "https://arxiv.org/abs/2608.24160"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-24160", "source": "claire_radar", "source_benchmark_id": "2608.24160", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": ["OmniJudge or OmniBias? Diagnosing Multimodal Judges through Balanced, Decoupled Lenses"], "annotation": "Mixed", "applicationDomains": ["General AI"], "area": "Multimodal", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2608.24160", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "unavailable"}, "attentionForecast": {"confidence": "Medium", "horizon": "7d", "reason": "The diagnostic approach to multimodal judge reliability addresses a growing concern, though lack of immediate public artifacts may temper initial engagement.", "score": 61}, "capabilities": [], "confidence": "Medium", "construction": "Aggregate Existing", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-27T04:18:56.318436Z", "inputHash": "f733bb137c345d4bddc33b33ef376129d7c3d1f1b92675a8198237abc4961f5b", "model": "deepseek-v4-pro"}, "curation": {"canonicalNameEvidence": "we introduce D3-Omni, a balanced and decoupled benchmark for diagnosing fine-grained multimodal understanding", "canonicalNameSource": "abstract", "decisionReason": "The benchmark provides a stable evaluation protocol with dual-balanced and decoupled design, but no explicit public release statement or artifact link is present, limiting immediate reuse.", "model": "deepseek-v4-pro", "reviewedAt": "2026-08-27T04:18:56.318436Z", "state": "ai-name-audit-deferred"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "D3-Omni diagnoses multimodal judge models across T2I, T2V, and TTS tasks with 53 balanced orthogonal binary dimensions and 10,671 samples that isolate specific capability failures.", "displayEligible": false, "domainCuration": {"method": "rules-v1", "state": "auto"}, "evaluationMode": "public_reusable", "evidence": {"reasonCodes": ["benchmark term in abstract", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "Motivated by this, we introduce D3-Omni, a balanced and decoupled benchmark for diagnosing fine-grained multimodal understanding, covering 53 orthogonal binary dimensions (17/22/14) and 10,671 samples (3,526/1,998/5,147) across the three tasks."}, "familyId": "bmf_664f4b52dc9b", "firstSeenAt": "2026-08-26", "heat": null, "id": "bm_omnijudge-or-omnibias-diagnosing-multimoda_74cb9dd9", "indexedAt": "2026-08-26T06:08:22.244942Z", "industrySectors": [], "links": {"code": null, "data": null, "paper": "https://arxiv.org/abs/2608.24160", "pdf": "https://arxiv.org/pdf/2608.24160", "project": null, "report": "https://arxiv.org/abs/2608.24160"}, "metrics": [], "motivation": "Multimodal understanding models that can jointly judge text-to-image (T2I), text-to-video (T2V) and text-to-speech (TTS) generation are increasingly used as \"OmniJudges\" for evaluation and automatic annotation.", "name": "D3-Omni", "oneLine": "D3-Omni diagnoses multimodal judge models across T2I, T2V, and TTS tasks with 53 balanced orthogonal binary dimensions and 10,671 samples that isolate specific capability failures.", "paperTitle": "OmniJudge or OmniBias? Diagnosing Multimodal Judges through Balanced, Decoupled Lenses", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.24160", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-08-26T06:08:22.244942Z"}, "publications": [], "ranking": {"90d": {"confidence": "Low", "coverage": 0.0, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": null, "value": null}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "rank": null, "score": null}, "method": "attention-ranking-v13", "rank": null, "score": null}}, "readiness": "Paper only", "recognitionConfidence": 0.6, "relation": "introduces", "releasedAt": "2026-08-25", "source": {"authors": ["Guangzheng Hu", "Ziyue Jiang", "Weixu Qiao", "Lixin Zhang", "Jianye Kang", "Yuru Wu", "Rong Bao", "Niantong Li", "Wei Wang", "Ziyi Cheng", "Xinfa Zhu", "HangRui Hu", "Ting He", "Bing Zhao", "Lin Qu", "Hu Wei", "Jin Xu"], "categories": ["cs.AI"], "id": "2608.24160", "title": "OmniJudge or OmniBias? Diagnosing Multimodal Judges through Balanced, Decoupled Lenses", "type": "arxiv", "url": "https://arxiv.org/abs/2608.24160"}, "sourceUpdatedAt": "2026-08-25T07:18:35Z", "topics": ["Multimodal"], "venueAttempts": [], "whyItMatters": "It exposes fine-grained blind spots in OmniJudges that aggregate accuracy hides, guiding more reliable automatic evaluation and annotation."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
