{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2606.07872", "kind": "paper", "url": "https://arxiv.org/abs/2606.07872"}, {"id": "gh:didizhu-judy/visualflip", "kind": "repo", "url": "https://github.com/didizhu-judy/VisualFLIP"}, {"kind": "website", "url": "https://didizhu-judy.github.io/VisualFLIP/"}], "authors": ["Didi Zhu", "Changrui Chen", "Stefanos Zafeiriou", "Jiankang Deng"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2606.07872", "source_url": "https://arxiv.org/abs/2606.07872"}, "categories": ["General AI", "Multimodal", "Reasoning"], "description": {"en": "VisualFLIP evaluates multimodal LLMs on visual reasoning with 1,374 images in paired perturbation tasks. Each pair has a fixed question but minimally changed visual evidence so the answer flips. Scoring uses pair accuracy and Collapse Rate to test evidence dependence in capabilities like cardinality, attribute, spatial, and logic."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2606.07872", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2606.07872", "title": "VisualFLIP"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["Imperial College London"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2606.07872", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2606.07872", "source_url": "https://openalex.org/W7164003103"}, "key": "claire-radar:2606.07872", "modality": null, "name": "VisualFLIP", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_visualflip_556bf533", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "eff8f335356e6ee9099c436282e3f0013ab30ff892b3f9e303d695f765bf2266", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2606.07872"}, "publisher": null, "released": "2026-06-05", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2606.07872", "source_url": "https://arxiv.org/abs/2606.07872"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2606-07872", "source": "claire_radar", "source_benchmark_id": "2606.07872", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Multimodal", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/didizhu-judy/VisualFLIP", "githubScope": "benchmark_repo", "githubStars": 1, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2606.07872", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "fresh"}, "capabilities": ["Reasoning"], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "f885a1b3364ce9ea25680e46db3dd46f70e2747bab2b7ef4b08044cddcf7878b", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "VisualFLIP evaluates multimodal LLMs on visual reasoning with 1,374 images in paired perturbation tasks. Each pair has a fixed question but minimally changed visual evidence so the answer flips. Scoring uses pair accuracy and Collapse Rate to test evidence dependence in capabilities like cardinality, attribute, spatial, and logic.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["exact coined title identity tied to benchmark evidence", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "We introduce VisualFLIP, a paired benchmark with 1,374 images arranged as same-question perturbation pairs across cardinality, attribute, spatial, and logic tasks."}, "familyId": "bmf_eb3ea10a5081", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 1, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "linked-from-project", "linkedUrl": "https://github.com/didizhu-judy/VisualFLIP", "url": "https://didizhu-judy.github.io/VisualFLIP/"}], "scope": "benchmark_repo", "url": "https://github.com/didizhu-judy/VisualFLIP", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_visualflip_556bf533", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": "https://github.com/didizhu-judy/VisualFLIP", "data": null, "hfPaper": "https://huggingface.co/papers/2606.07872", "paper": "https://arxiv.org/abs/2606.07872", "pdf": "https://arxiv.org/pdf/2606.07872", "project": "https://didizhu-judy.github.io/VisualFLIP/", "report": "https://arxiv.org/abs/2606.07872"}, "metrics": [], "motivation": "When a multimodal large language model answers a visual reasoning question correctly, is the prediction actually supported by the task-critical visual evidence?", "name": "VisualFLIP", "oneLine": "VisualFLIP evaluates multimodal LLMs on visual reasoning with 1,374 images in paired perturbation tasks. Each pair has a fixed question but minimally changed visual evidence so the answer flips. Scoring uses pair accuracy and Collapse Rate to test evidence dependence in capabilities like cardinality, attribute, spatial, and logic.", "paperTitle": "VisualFLIP: Do Predictions Depend on Task-Critical Visual Evidence in Multimodal Reasoning?", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2606.07872", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "VisualFLIP team", "organizationType": "academic-lab", "role": "benchmark-publisher", "sourceUrl": "https://didizhu-judy.github.io/VisualFLIP/"}], "ranking": {}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-06-05", "source": {"authors": ["Didi Zhu", "Changrui Chen", "Stefanos Zafeiriou", "Jiankang Deng"], "categories": ["cs.CV"], "id": "2606.07872", "title": "VisualFLIP: Do Predictions Depend on Task-Critical Visual Evidence in Multimodal Reasoning?", "type": "arxiv", "url": "https://arxiv.org/abs/2606.07872"}, "sourceUpdatedAt": "2026-06-05T22:06:07Z", "topics": ["Multimodal", "Reasoning"], "venueAttempts": [], "whyItMatters": "Accuracy alone can hide flawed reasoning. VisualFLIP exposes whether models truly rely on task-critical visual changes, distinguishing robust grounding from guesswork. This helps select models for high-stakes visual tasks."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
