{"record": {"aliases": ["A Unified Issue Resolution Benchmark for Requirement Clarification, Planning, and Code Generation for Coding Agents"], "artifacts": [{"id": "arxiv:2608.09072", "kind": "paper", "url": "https://arxiv.org/abs/2608.09072"}, {"id": "gh:xin-zhou-smu/swe-rpg-bench", "kind": "repo", "url": "https://github.com/Xin-Zhou-smu/SWE-RPG-Bench"}], "authors": ["Xin Zhou", "Chun Yong Chong", "Kisub Kim", "Yun Peng", "Rui Shu", "Zihan Wu", "Xu Han", "Guowen Yuan", "Zeyang Zhuang", "Jounghoon Kim", "Jeongjin Ju", "Seongmin Ju", "Taein Yoon", "David Lo"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.09072", "source_url": "https://arxiv.org/abs/2608.09072"}, "categories": ["Code", "Code & Software", "Code generation", "Planning", "Software & AI Compute"], "description": {"en": "A repository-level dataset and protocol evaluating issue resolution through requirement clarification, implementation planning, code generation, and artifact submission, with 163 tasks across 31 Python and Java repositories and validated intermediate ground-truth references."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.09072", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.09072", "title": "SWE-RPG"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["Singapore Management University", "Monash University Malaysia", "Daegu Gyeongbuk Institute of Science and Technology", "Chinese University of Hong Kong", "North Carolina State University", "City University of Hong Kong", "Hong Kong University of Science and Technology", "University of Hong Kong"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2608.09072", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2608.09072", "source_url": "https://openalex.org/W7202166648"}, "key": "claire-radar:2608.09072", "modality": null, "name": "SWE-RPG", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "Medium", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "true", "origin_record_id": "bm_a-unified-issue-resolution-benchmark-for-r_de70fd7d", "origin_source": "arxiv", "recognition_confidence": "0.7", "record_sha256": "53d4f32dee615a313781bd4c4f001d2bedaf32d766c2b6e51cf2bae9cd7fc036", "relation": "introduces", "review_model": "deepseek-v4-pro", "review_state": "ai-reviewed", "reviewed_at": "2026-09-02T08:54:07.291235Z", "source_url": "https://arxiv.org/abs/2608.09072"}, "publisher": null, "released": "2026-08-10", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.09072", "source_url": "https://arxiv.org/abs/2608.09072"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-09072", "source": "claire_radar", "source_benchmark_id": "2608.09072", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": ["A Unified Issue Resolution Benchmark for Requirement Clarification, Planning, and Code Generation for Coding Agents"], "annotation": "Unknown", "applicationDomains": ["Software & AI Compute"], "area": "Code & Software", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/Xin-Zhou-smu/SWE-RPG-Bench", "githubScope": "benchmark_repo", "githubStars": 3, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 0, "hfPaperUrl": "https://huggingface.co/papers/2608.09072", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}}, "status": "fresh"}, "attentionForecast": {"confidence": "Medium", "horizon": "7d", "reason": "The benchmark addresses a widely recognized pain point in coding-agent evaluation with a sizeable multi-repository task set and immediate published code, so repository-modification researchers are likely to inspect and discuss it quickly.", "score": 75}, "capabilities": ["Planning", "Code generation"], "confidence": "Medium", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-09-02T08:54:07.291235Z", "inputHash": "049bd249c5ad907dd040f80eba0333bd24a30cd893f763d99cd46df294543dce", "model": "deepseek-v4-pro"}, "curation": {"canonicalNameEvidence": "we introduce SWE-RPG, a repository-level benchmark", "canonicalNameSource": "abstract", "decisionReason": "The paper explicitly introduces SWE-RPG as a benchmark, provides a stable scoring contract combining executable patch evaluation with validated intermediate references, and states that benchmark data and evaluation code are available at the public GitHub repository.", "model": "deepseek-v4-pro", "reviewedAt": "2026-09-02T08:54:07.291235Z", "state": "ai-reviewed"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "A repository-level dataset and protocol evaluating issue resolution through requirement clarification, implementation planning, code generation, and artifact submission, with 163 tasks across 31 Python and Java repositories and validated intermediate ground-truth references.", "displayEligible": true, "domainCuration": {"method": "rules-v1", "state": "auto"}, "evaluationMode": "public_reusable", "evidence": {"reasonCodes": ["benchmark term in abstract", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "To address this gap, we introduce SWE-RPG, a repository-level benchmark that combines executable patch evaluation with validated ground-truth references (GTs) for (1) Requirement Clarification and (2) Implementation Planning."}, "familyId": "bmf_6c301d49c3b1", "firstSeenAt": "2026-08-19", "heat": null, "id": "bm_a-unified-issue-resolution-benchmark-for-r_de70fd7d", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": ["Software & Cloud"], "links": {"code": "https://github.com/Xin-Zhou-smu/SWE-RPG-Bench", "data": null, "paper": "https://arxiv.org/abs/2608.09072", "pdf": "https://arxiv.org/pdf/2608.09072", "project": null, "report": "https://arxiv.org/abs/2608.09072"}, "metrics": [], "motivation": "Large language model-powered coding agents are increasingly used to modify existing code repositories, for example, by adding features or fixing bugs.", "name": "SWE-RPG", "oneLine": "A repository-level dataset and protocol evaluating issue resolution through requirement clarification, implementation planning, code generation, and artifact submission, with 163 tasks across 31 Python and Java repositories and validated intermediate ground-truth references.", "paperTitle": "A Unified Issue Resolution Benchmark for Requirement Clarification, Planning, and Code Generation for Coding Agents", "primaryDomain": "Software & AI Compute", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.09072", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-08-19T11:05:13.395391Z"}, "publications": [], "publishers": [{"name": "SMU", "organizationType": "academic-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/Xin-Zhou-smu/SWE-RPG-Bench"}], "ranking": {"90d": {"confidence": "Medium", "coverage": 0.7, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.18504537701418716, "value": 3}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.0, "value": 0}}, "confidence": "Medium", "coverage": 0.7, "rank": 365, "score": 25}, "method": "attention-ranking-v13", "rank": 365, "score": 25}}, "readiness": "Runnable", "recognitionConfidence": 0.7, "relation": "introduces", "releasedAt": "2026-08-10", "source": {"authors": ["Xin Zhou", "Chun Yong Chong", "Kisub Kim", "Yun Peng", "Rui Shu", "Zihan Wu", "Xu Han", "Guowen Yuan", "Zeyang Zhuang", "Jounghoon Kim", "Jeongjin Ju", "Seongmin Ju", "Taein Yoon", "and David Lo"], "categories": ["cs.SE", "cs.AI"], "id": "2608.09072", "title": "A Unified Issue Resolution Benchmark for Requirement Clarification, Planning, and Code Generation for Coding Agents", "type": "arxiv", "url": "https://arxiv.org/abs/2608.09072"}, "sourceUpdatedAt": "2026-08-10T03:22:04Z", "topics": ["Code"], "venueAttempts": [], "whyItMatters": "Existing repository benchmarks only check whether final patches pass tests, concealing where coding agents fail; intermediate ground truths enable diagnosing implicit requirement recovery as the main bottleneck and guide improvements in planning and code generation for iterative repository editing."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
