{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2606.12344", "kind": "paper", "url": "https://arxiv.org/abs/2606.12344"}, {"id": "gh:opensquilla/claw-swe-bench", "kind": "repo", "url": "https://github.com/opensquilla/claw-swe-bench"}, {"id": "hf:tokenrhythm/claw-swe-bench", "kind": "dataset", "url": "https://huggingface.co/datasets/TokenRhythm/Claw-SWE-Bench"}], "authors": ["Mengyu Zheng", "Kai Han", "Boxun Li", "Haiyang Xu", "Yuchuan Tian", "Wei He", "Hang Zhou", "Jianyuan Guo", "Hailin Hu", "Lin Ma", "Chao Xu", "Guohao Dai", "Lixue Xia", "Yunchao Wei", "Yunhe Wang", "Yu Wang"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2606.12344", "source_url": "https://arxiv.org/abs/2606.12344"}, "categories": ["Agents", "General AI", "Language & Knowledge"], "description": {"en": "Claw-SWE-Bench is a multilingual SWE-bench-style benchmark and adapter protocol for comparing agent harnesses on coding tasks, with 350 GitHub issue-resolution instances across 8 languages and 43 repositories. Score is Pass@1 on patch correctness."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2606.12344", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2606.12344", "title": "Claw-SWE-Bench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["Peking University", "City University of Hong Kong", "The Doe Fund", "Shanghai Jiao Tong University", "Beijing Jiaotong University", "Tsinghua University"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2606.12344", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2606.12344", "source_url": "https://openalex.org/W7164437655"}, "key": "claire-radar:2606.12344", "modality": null, "name": "Claw-SWE-Bench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_claw-swe-bench_ea69cc9d", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "76f04a701bc2a4b027b39e1c94f8f294e4ad99fdcbb65fb2d0df2c9e7a0fa86d", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2606.12344"}, "publisher": null, "released": "2026-06-10", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2606.12344", "source_url": "https://arxiv.org/abs/2606.12344"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2606-12344", "source": "claire_radar", "source_benchmark_id": "2606.12344", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/opensquilla/claw-swe-bench", "githubScope": "benchmark_repo", "githubStars": 110, "hfDailySubmittedAt": null, "hfDataset": "TokenRhythm/Claw-SWE-Bench", "hfDatasetDownloads": 3000, "hfDatasetLikes": 8, "hfPaperUpvotes": 40, "hfPaperUrl": "https://huggingface.co/papers/2606.12344", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-24", "state": "stale"}, "hfDatasetLikes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-24", "state": "stale"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-10-01", "state": "stale"}}, "status": "fresh"}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "f8bb0766fa75a120dfa40c1479579a5d65c458ccac51b1b83bcc162300dbf3fb", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "Claw-SWE-Bench is a multilingual SWE-bench-style benchmark and adapter protocol for comparing agent harnesses on coding tasks, with 350 GitHub issue-resolution instances across 8 languages and 43 repositories. Score is Pass@1 on patch correctness.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "We introduce Claw-SWE-Bench, a multilingual SWE-bench-style benchmark and adapter protocol that makes heterogeneous agent harnesses, or claws, comparable under fair settings including a fixed prompt, runtime budget, workspace contract, patch extraction procedure, and evaluator."}, "familyId": "bmf_32be4dea95f8", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 6, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "existing-code-link", "linkedUrl": "https://github.com/opensquilla/claw-swe-bench", "url": "https://github.com/opensquilla/claw-swe-bench"}, {"basis": "existing-metric-association", "linkedUrl": "https://github.com/opensquilla/claw-swe-bench", "url": "https://huggingface.co/papers/2606.12344"}, {"basis": "huggingface-paper-association", "linkedUrl": "https://github.com/opensquilla/claw-swe-bench", "url": "https://huggingface.co/api/papers/2606.12344"}, {"basis": "linked-from-report", "linkedUrl": "https://github.com/opensquilla/claw-swe-bench", "url": "https://arxiv.org/abs/2606.12344"}], "scope": "benchmark_repo", "url": "https://github.com/opensquilla/claw-swe-bench", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_claw-swe-bench_ea69cc9d", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": "https://github.com/opensquilla/claw-swe-bench", "data": "https://huggingface.co/datasets/TokenRhythm/Claw-SWE-Bench", "hfPaper": "https://huggingface.co/papers/2606.12344", "paper": "https://arxiv.org/abs/2606.12344", "pdf": "https://arxiv.org/pdf/2606.12344", "project": null, "report": "https://arxiv.org/abs/2606.12344"}, "metrics": [], "motivation": "General-purpose agents such as OpenClaw are increasingly used as autonomous tool users, but their coding ability is difficult to measure under SWE-bench: a generic agent does not by itself satisfy the clean Docker workspace, patch, and prediction contract required for scoring.", "name": "Claw-SWE-Bench", "oneLine": "Claw-SWE-Bench is a multilingual SWE-bench-style benchmark and adapter protocol for comparing agent harnesses on coding tasks, with 350 GitHub issue-resolution instances across 8 languages and 43 repositories. Score is Pass@1 on patch correctness.", "paperTitle": "Claw-SWE-Bench: A Benchmark for Evaluating OpenClaw-style Agent Harnesses on Coding Tasks", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2606.12344", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "TokenRhythm Technologies", "organizationType": "company-research-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/opensquilla/claw-swe-bench"}], "ranking": {}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-06-10", "source": {"authors": ["Mengyu Zheng", "Kai Han", "Boxun Li", "Haiyang Xu", "Yuchuan Tian", "Wei He", "Hang Zhou", "Jianyuan Guo", "Hailin Hu", "Lin Ma", "Chao Xu", "Guohao Dai", "Lixue Xia", "Yunchao Wei", "Yunhe Wang", "Yu Wang"], "categories": ["cs.LG", "cs.CL"], "id": "2606.12344", "title": "Claw-SWE-Bench: A Benchmark for Evaluating OpenClaw-style Agent Harnesses on Coding Tasks", "type": "arxiv", "url": "https://arxiv.org/abs/2606.12344"}, "sourceUpdatedAt": "2026-06-10T17:16:23Z", "topics": ["Agents"], "venueAttempts": [], "whyItMatters": "Enables fair comparison of autonomous coding agents by treating harness and cost as first-class evaluation axes. Useful for developers and researchers building general-purpose coding agents."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
