{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2608.11469", "kind": "paper", "url": "https://arxiv.org/abs/2608.11469"}], "authors": ["Jeremy Spence", "Nicholas Assaderaghi", "Feng Xiao", "Jinhao Zhu", "Nikil Ravi", "Xiangyu Qi", "Matthew Jagielski", "Raluca Ada Popa", "Eric Wallace", "Guannan Wei", "Yangruibo Ding", "Zhuo Zhang"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.11469", "source_url": "https://arxiv.org/abs/2608.11469"}, "categories": ["Cybersecurity", "Language & Knowledge", "cs.CR"], "description": {"en": "SRE-Bench evaluates AI agents on reverse engineering of binaries compiled from 19 private real-world-scale C programs (average 16.9K LoC) with 44 anti-analysis primitives, yielding 262 binary instances and 1572 deterministically graded tasks."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.11469", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.11469", "title": "SRE-Bench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["Columbia University", "University of California, Berkeley", "Tufts University", "University of California, Los Angeles"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2608.11469", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2608.11469", "source_url": "https://openalex.org/W7202393961"}, "key": "claire-radar:2608.11469", "modality": null, "name": "SRE-Bench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_sre-bench_d394a59b", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "40bcc56174316063200df9dd9b75cc9df6fd520a3e3aff2d1dcdd1d1ec55976c", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2608.11469"}, "publisher": null, "released": "2026-08-11", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.11469", "source_url": "https://arxiv.org/abs/2608.11469"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-11469", "source": "claire_radar", "source_benchmark_id": "2608.11469", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["Cybersecurity"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 2, "hfPaperUrl": "https://huggingface.co/papers/2608.11469", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}}, "status": "fresh"}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "2aeed24a8279cc24f3368bd4c0dda9ef029042a24563a915d52b0627e3d81ba2", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "SRE-Bench evaluates AI agents on reverse engineering of binaries compiled from 19 private real-world-scale C programs (average 16.9K LoC) with 44 anti-analysis primitives, yielding 262 binary instances and 1572 deterministically graded tasks.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["exact named benchmark artifact released in abstract", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "To this end, we introduce SRE-Bench, the first realistic, contamination-free RE benchmark."}, "familyId": "bmf_d2aa6d139e85", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_sre-bench_d394a59b", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": ["Cybersecurity"], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2608.11469", "paper": "https://arxiv.org/abs/2608.11469", "pdf": "https://arxiv.org/pdf/2608.11469", "project": null, "report": "https://arxiv.org/abs/2608.11469"}, "metrics": [], "motivation": "AI agents are rapidly improving in cybersecurity capabilities when the source code is available for analysis, yet much of the software most consequential to cybersecurity, including malware, firmware, and proprietary applications, is available only as binaries.", "name": "SRE-Bench", "oneLine": "SRE-Bench evaluates AI agents on reverse engineering of binaries compiled from 19 private real-world-scale C programs (average 16.9K LoC) with 44 anti-analysis primitives, yielding 262 binary instances and 1572 deterministically graded tasks.", "paperTitle": "The Next Challenge for Agentic Cybersecurity: A Realistic, Contamination-Free Reverse Engineering Benchmark", "primaryDomain": "Cybersecurity", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.11469", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {"90d": {"confidence": "Low", "coverage": 0.15, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": null, "value": null}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.19411886932144953, "value": 2}}, "confidence": "Low", "coverage": 0.15, "rank": null, "score": 45}, "method": "attention-ranking-v13", "rank": null, "score": 45}}, "readiness": "Paper only", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-08-11", "source": {"authors": ["Jeremy Spence", "Nicholas Assaderaghi", "Jinhao Zhu", "Nikil Ravi", "Raluca Ada Popa", "Guannan Wei", "Yangruibo Ding", "Zhuo Zhang"], "categories": ["cs.CR", "cs.AI", "cs.SE"], "id": "2608.11469", "title": "The Next Challenge for Agentic Cybersecurity: A Realistic, Contamination-Free Reverse Engineering Benchmark", "type": "arxiv", "url": "https://arxiv.org/abs/2608.11469"}, "sourceUpdatedAt": "2026-08-11T22:14:57Z", "topics": ["cs.CR"], "venueAttempts": [], "whyItMatters": "Existing benchmarks for agentic cybersecurity miss either contamination control or realistic scale. SRE-Bench addresses this gap by combining private, real-world-scale binaries with deterministic grading, enabling reliable measurement of agent performance in binary analysis and highlighting the gap between source-code and binary security capabilities."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
