{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2607.10400", "kind": "paper", "url": "https://arxiv.org/abs/2607.10400"}, {"id": "gh:servicenow/synthdocbench", "kind": "repo", "url": "https://github.com/ServiceNow/SynthDocBench"}], "authors": ["Abhigya Verma", "Khyati Mahajan", "Amit Kumar Saha", "Shruthan Radhakrishna", "Sagar Davasam", "Vikas Yadav", "Sai Rajeswar Mudumba"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2607.10400", "source_url": "https://arxiv.org/abs/2607.10400"}, "categories": ["General AI", "Long Context", "Vision & 3D"], "description": {"en": "SynthDocBench evaluates vision-language models on 1,788 questions over 200 synthetic long-context documents (avg. 51.1 pages) with 3,340 charts. It varies document length, layout archetype, modality composition, and question type as independent controlled factors, across chart, cross-modal, and complex subsets with deterministic ground truth."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2607.10400", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2607.10400", "title": "SynthDocBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["Université de Montréal"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2607.10400", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2607.10400", "source_url": "https://openalex.org/W7168273980"}, "key": "claire-radar:2607.10400", "modality": null, "name": "SynthDocBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "Medium", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_synthdocbench_7af1a05d", "origin_source": "arxiv", "recognition_confidence": "0.85", "record_sha256": "be1d74e36fbfb76448b8f6501a03938bca0625eba23f5f9b2f27de6a4a263c94", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2607.10400"}, "publisher": null, "released": "2026-07-11", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2607.10400", "source_url": "https://arxiv.org/abs/2607.10400"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2607-10400", "source": "claire_radar", "source_benchmark_id": "2607.10400", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Machine Generated", "applicationDomains": ["General AI"], "area": "Vision & 3D", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/ServiceNow/SynthDocBench", "githubScope": "benchmark_repo", "githubStars": 9, "hfDailySubmittedAt": "2026-07-15T00:00:00.000Z", "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 68, "hfPaperUrl": "https://huggingface.co/papers/2607.10400", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}}, "status": "fresh"}, "capabilities": [], "confidence": "Medium", "construction": "Original Synthetic", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "a77aa761d725adc596250e12d35b6b03c1d0b02ed127e66a50dd9ccf9c1383c2", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "SynthDocBench evaluates vision-language models on 1,788 questions over 200 synthetic long-context documents (avg. 51.1 pages) with 3,340 charts. It varies document length, layout archetype, modality composition, and question type as independent controlled factors, across chart, cross-modal, and complex subsets with deterministic ground truth.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence"], "snippet": "We introduce SynthDocBench, a fully synthetic benchmark for long-context visual document understanding that systematically controls factors including document length, layout structure, modality composition, and question type."}, "familyId": "bmf_3a3c4b0877bd", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "existing-code-link", "linkedUrl": "https://github.com/ServiceNow/SynthDocBench", "url": "https://github.com/ServiceNow/SynthDocBench"}, {"basis": "existing-metric-association", "linkedUrl": "https://github.com/ServiceNow/SynthDocBench", "url": "https://huggingface.co/papers/2607.10400"}], "scope": "benchmark_repo", "url": "https://github.com/ServiceNow/SynthDocBench", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_synthdocbench_7af1a05d", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": "https://github.com/ServiceNow/SynthDocBench", "data": null, "hfPaper": "https://huggingface.co/papers/2607.10400", "paper": "https://arxiv.org/abs/2607.10400", "pdf": "https://arxiv.org/pdf/2607.10400", "project": null, "report": "https://arxiv.org/abs/2607.10400"}, "metrics": [], "motivation": "Vision language models (VLMs) have achieved strong performance on visual document understanding benchmarks such as DocVQA, ChartQA, and MMLongBench-Doc.", "name": "SynthDocBench", "oneLine": "SynthDocBench evaluates vision-language models on 1,788 questions over 200 synthetic long-context documents (avg. 51.1 pages) with 3,340 charts. It varies document length, layout archetype, modality composition, and question type as independent controlled factors, across chart, cross-modal, and complex subsets with deterministic ground truth.", "paperTitle": "SynthDocBench: Controlled Benchmark for Long-Context Visual Document Understanding", "primaryDomain": "General AI", "publication": {"evidence": "29 Pages, 27 Tables, 13 Figures, Accepted at COLM 2026", "evidenceLevel": "author-claim", "evidenceUrl": "https://arxiv.org/abs/2607.10400", "source": "arxiv-comments", "status": "acceptance_claimed", "venue": "COLM 2026", "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "ServiceNow AI", "organizationType": "company-research-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/ServiceNow/SynthDocBench"}], "ranking": {"90d": {"confidence": "Medium", "coverage": 0.7, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.30735371836622627, "value": 9}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.7481437953470013, "value": 68}}, "confidence": "Medium", "coverage": 0.7, "rank": 121, "score": 43}, "method": "attention-ranking-v13", "rank": 121, "score": 43}}, "readiness": "Runnable", "recognitionConfidence": 0.85, "relation": "introduces", "releasedAt": "2026-07-11", "source": {"authors": ["Abhigya Verma", "Khyati Mahajan", "Amit Kumar Saha", "Shruthan Radhakrishna", "Sagar Davasam", "Vikas Yadav", "Sai Rajeswar Mudumba"], "categories": ["cs.CV", "cs.AI"], "id": "2607.10400", "title": "SynthDocBench: Controlled Benchmark for Long-Context Visual Document Understanding", "type": "arxiv", "url": "https://arxiv.org/abs/2607.10400"}, "sourceUpdatedAt": "2026-07-11T17:02:49Z", "topics": ["Long Context"], "venueAttempts": [{"decisionRaw": "29 Pages, 27 Tables, 13 Figures, Accepted at COLM 2026", "evidence": [{"level": "author-claim", "observedAt": "2026-10-14T01:46:58.058199Z", "rawValue": "29 Pages, 27 Tables, 13 Figures, Accepted at COLM 2026", "sourceType": "arxiv-comments", "sourceUrl": "https://arxiv.org/abs/2607.10400"}], "reviewStatus": "accepted", "venueName": "COLM 2026"}], "whyItMatters": "Existing document benchmarks confound length, layout, and modality, obscuring specific model failure causes. SynthDocBench provides controlled attribution of failures, revealing sharp degradation with document length, positional sensitivity in the middle third, and breakdown of chart comprehension in long documents, which is valuable for targeted model improvement."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
