{"record": {"aliases": ["SA-Bench"], "artifacts": [{"id": "arxiv:2608.24252", "kind": "paper", "url": "https://arxiv.org/abs/2608.24252"}], "authors": ["Xue Hu", "Zewei Pan", "Zeli Su", "Zhou Liu", "Wentao Zhang"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.24252", "source_url": "https://arxiv.org/abs/2608.24252"}, "categories": ["General AI", "Language & Knowledge", "Safety"], "description": {"en": "SA-Bench (SemanticAlign-Bench) evaluates semantic alignment in LLM-based paper reproduction across 30 papers from top conferences. It decomposes paper specifications into atomic verifiable claims (SAUs) and evaluates repositories along four diagnostic dimensions (numerical, methodological, protocol, ordering drift). Includes 1,491 SAUs across five ML domains and evaluates 12 generator configurations."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.24252", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.24252", "title": "SA-Bench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2608.24252", "modality": null, "name": "SA-Bench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "true", "origin_record_id": "bm_sa-bench_734de9b3", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "ac816fc9bc56f89332d7c6ddc20ff51283b7c9aa980eaed00e0d5a5336aff6bd", "relation": "introduces", "review_model": "deepseek-v4-pro", "review_state": "ai-reviewed", "reviewed_at": "2026-08-27T04:18:56.318436Z", "source_url": "https://arxiv.org/abs/2608.24252"}, "publisher": null, "released": "2026-08-25", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.24252", "source_url": "https://arxiv.org/abs/2608.24252"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-24252", "source": "claire_radar", "source_benchmark_id": "2608.24252", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": ["SA-Bench"], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-08-26", "attemptedAt": "2026-08-26T11:32:18.902760Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2608.24252", "observedAt": "2026-08-24T11:31:30.653116Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-08-26T11:32:18.902760Z", "state": "unavailable"}}, "status": "unavailable"}, "attentionForecast": {"confidence": "Medium", "horizon": "7d", "reason": "Niche but relevant to AI for science and code generation; likely to engage reproducibility-focused researchers.", "score": 60}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-27T04:22:25.189955Z", "inputHash": "1a6f709d680ba9855c1761a223024fd978e786b6cbd1bd3833d2bd36fc399159", "model": "deepseek-v4-flash"}, "curation": {"canonicalNameEvidence": "We introduce SemanticAlign-Bench(SA-Bench), a diagnostic benchmark", "canonicalNameSource": "abstract", "decisionReason": "The benchmark is formally named, has a stable scoring contract via SAU evaluation, and states that the benchmark, annotations, and pipeline are publicly available, though no direct artifact link is included.", "model": "deepseek-v4-pro", "reviewedAt": "2026-08-27T04:18:56.318436Z", "state": "ai-reviewed"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "SA-Bench (SemanticAlign-Bench) evaluates semantic alignment in LLM-based paper reproduction across 30 papers from top conferences. It decomposes paper specifications into atomic verifiable claims (SAUs) and evaluates repositories along four diagnostic dimensions (numerical, methodological, protocol, ordering drift). Includes 1,491 SAUs across five ML domains and evaluates 12 generator configurations.", "displayEligible": true, "domainCuration": {"method": "rules-v1", "state": "auto"}, "evaluationMode": "public_reusable", "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "We introduce SemanticAlign-Bench(SA-Bench), a diagnostic benchmark covering 30 papers from ICLR, ICML and NeurIPS 2025."}, "familyId": "bmf_4562cb20b278", "firstSeenAt": "2026-08-26", "heat": null, "id": "bm_sa-bench_734de9b3", "indexedAt": "2026-08-26T06:08:22.244942Z", "industrySectors": [], "links": {"code": null, "data": null, "paper": "https://arxiv.org/abs/2608.24252", "pdf": "https://arxiv.org/pdf/2608.24252", "project": null, "report": "https://arxiv.org/abs/2608.24252"}, "metrics": [], "motivation": "LLM agents can generate paper reproduction code, yet often produce scientifically unfaithful implementations.", "name": "SA-Bench", "oneLine": "SA-Bench (SemanticAlign-Bench) evaluates semantic alignment in LLM-based paper reproduction across 30 papers from top conferences. It decomposes paper specifications into atomic verifiable claims (SAUs) and evaluates repositories along four diagnostic dimensions (numerical, methodological, protocol, ordering drift). Includes 1,491 SAUs across five ML domains and evaluates 12 generator configurations.", "paperTitle": "SA-Bench: Evaluating Semantic Alignment in LLM-Based Paper Reproduction", "primaryDomain": "General AI", "publication": {"evidence": "Accepted to Findings of EMNLP 2026", "evidenceLevel": "author-claim", "evidenceUrl": "https://arxiv.org/abs/2608.24252", "source": "arxiv-comments", "status": "acceptance_claimed", "venue": "Findings of EMNLP 2026", "verifiedAt": "2026-08-26T06:08:22.244942Z"}, "publications": [], "ranking": {"30d": {"confidence": "Low", "coverage": 0.0, "growth": {"components": {"githubStars": {"percentile": null, "value": null}, "hfDatasetDownloads": {"percentile": null, "value": null}, "hfPaperUpvotes": {"percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 30}, "level": {"components": {"githubStars": {"percentile": null, "value": null}, "hfDatasetDownloads": {"percentile": null, "value": null}, "hfPaperUpvotes": {"percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "rank": null, "score": null}, "method": "attention-ranking-v12", "rank": null, "score": null}, "90d": {"confidence": "Low", "coverage": 0.0, "growth": {"components": {"githubStars": {"percentile": null, "value": null}, "hfDatasetDownloads": {"percentile": null, "value": null}, "hfPaperUpvotes": {"percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"percentile": null, "value": null}, "hfDatasetDownloads": {"percentile": null, "value": null}, "hfPaperUpvotes": {"percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "rank": null, "score": null}, "method": "attention-ranking-v12", "rank": null, "score": null}, "today": {"confidence": "Low", "coverage": 0.0, "growth": {"components": {"githubStars": {"percentile": null, "value": null}, "hfDatasetDownloads": {"percentile": null, "value": null}, "hfPaperUpvotes": {"percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": 2, "historyDate": "2026-08-24", "rank": null, "score": null, "windowDays": 1}, "level": {"components": {"githubStars": {"percentile": null, "value": null}, "hfDatasetDownloads": {"percentile": null, "value": null}, "hfPaperUpvotes": {"percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "rank": null, "score": null}, "method": "attention-ranking-v12", "rank": null, "score": null}}, "readiness": "Paper only", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-08-25", "source": {"authors": ["Xue Hu", "Zewei Pan", "Zeli Su", "Zhou Liu and Wentao Zhang"], "categories": ["cs.AI", "cs.SE"], "id": "2608.24252", "title": "SA-Bench: Evaluating Semantic Alignment in LLM-Based Paper Reproduction", "type": "arxiv", "url": "https://arxiv.org/abs/2608.24252"}, "sourceUpdatedAt": "2026-08-25T08:48:48Z", "topics": ["Safety"], "venueAttempts": [{"decisionRaw": "Accepted to Findings of EMNLP 2026", "evidence": [{"level": "author-claim", "observedAt": "2026-08-26T06:08:22.244942Z", "rawValue": "Accepted to Findings of EMNLP 2026", "sourceType": "arxiv-comments", "sourceUrl": "https://arxiv.org/abs/2608.24252"}], "reviewStatus": "accepted", "venueName": "Findings of EMNLP 2026"}], "whyItMatters": "LLM agents generating code for paper reproduction often produce semantically unfaithful implementations. SA-Bench provides a diagnostic framework to measure semantic drift, revealing that current agents struggle with faithful implementation, guiding development of better scaffolding."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
