{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2606.01629", "kind": "paper", "url": "https://arxiv.org/abs/2606.01629"}, {"id": "gh:cjj826/longjudgebench", "kind": "repo", "url": "https://github.com/cjj826/LongJudgeBench"}], "authors": ["Junjie Chen", "Yuxi Dong", "Haitao Li", "Weihang Su", "Yujia Zhou", "Min Zhang", "Yiqun Liu", "Qingyao Ai"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2606.01629", "source_url": "https://arxiv.org/abs/2606.01629"}, "categories": ["General AI", "Language & Knowledge", "cs.CL"], "description": {"en": "LongJudgeBench evaluates LLM-as-a-judge performance on long-form outputs across six datasets covering pointwise, pairwise, and listwise protocols, with bilingual tasks and multiple prompt variants. It measures agreement with human judgments using accuracy, Spearman, and Kendall's tau."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2606.01629", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2606.01629", "title": "LongJudgeBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2606.01629", "modality": null, "name": "LongJudgeBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_longjudgebench_1675bac5", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "ab33d735c6f07d3b8535891264339d6f8aeced5ec26c78a6cb7dc725c2dd3d7d", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2606.01629"}, "publisher": null, "released": "2026-06-01", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2606.01629", "source_url": "https://arxiv.org/abs/2606.01629"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2606-01629", "source": "claire_radar", "source_benchmark_id": "2606.01629", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/cjj826/LongJudgeBench", "githubScope": "benchmark_repo", "githubStars": 5, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2606.01629", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "fresh"}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "9c4ac4a6122fbef7436d7a2b26ecff815871da43423d65c19bf5b4396ded1e84", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "LongJudgeBench evaluates LLM-as-a-judge performance on long-form outputs across six datasets covering pointwise, pairwise, and listwise protocols, with bilingual tasks and multiple prompt variants. It measures agreement with human judgments using accuracy, Spearman, and Kendall's tau.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["exact named benchmark artifact released in abstract", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "In this work, we introduce LongJudgeBench, a comprehensive benchmark for evaluating LLM judges on long-form outputs across diverse real-world scenarios and judging protocols."}, "familyId": "bmf_36c4ccde3d0e", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "existing-code-link", "linkedUrl": "https://github.com/cjj826/LongJudgeBench", "url": "https://github.com/cjj826/LongJudgeBench"}, {"basis": "existing-metric-association", "linkedUrl": "https://github.com/cjj826/LongJudgeBench", "url": "https://huggingface.co/papers/2606.01629"}, {"basis": "linked-from-report", "linkedUrl": "https://github.com/cjj826/LongJudgeBench", "url": "https://arxiv.org/abs/2606.01629"}, {"basis": "linked-from-report", "linkedUrl": "https://github.com/cjj826/LongJudgeBench.", "url": "https://arxiv.org/abs/2606.01629"}], "scope": "benchmark_repo", "url": "https://github.com/cjj826/LongJudgeBench", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_longjudgebench_1675bac5", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": "https://github.com/cjj826/LongJudgeBench", "data": null, "hfPaper": "https://huggingface.co/papers/2606.01629", "paper": "https://arxiv.org/abs/2606.01629", "pdf": "https://arxiv.org/pdf/2606.01629", "project": null, "report": "https://arxiv.org/abs/2606.01629"}, "metrics": [], "motivation": "As large language models (LLMs) are increasingly used for long-form generation, reliably evaluating long-form outputs has become a critical challenge.", "name": "LongJudgeBench", "oneLine": "LongJudgeBench evaluates LLM-as-a-judge performance on long-form outputs across six datasets covering pointwise, pairwise, and listwise protocols, with bilingual tasks and multiple prompt variants. It measures agreement with human judgments using accuracy, Spearman, and Kendall's tau.", "paperTitle": "Benchmarking LLM-as-a-Judge for Long-Form Output Evaluation", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2606.01629", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "LongJudgeBench Team", "organizationType": "academic-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/cjj826/LongJudgeBench"}], "ranking": {}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-06-01", "source": {"authors": ["Junjie Chen", "Yuxi Dong", "Haitao Li", "Weihang Su", "Yujia Zhou", "Min Zhang", "Yiqun Liu", "Qingyao Ai"], "categories": ["cs.CL"], "id": "2606.01629", "title": "Benchmarking LLM-as-a-Judge for Long-Form Output Evaluation", "type": "arxiv", "url": "https://arxiv.org/abs/2606.01629"}, "sourceUpdatedAt": "2026-08-10T07:32:34Z", "topics": ["cs.CL"], "venueAttempts": [], "whyItMatters": "Existing meta-evaluation benchmarks focus on short-form outputs, leaving a gap for long-form evaluation. This benchmark provides a standardized way to assess judge reliability across diverse scenarios, helping practitioners select or develop judges for long-form tasks where current models show instability."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
