{"record": {"aliases": ["Kotlin SWE Benchmark"], "artifacts": [{"id": "gh:kotlin/kotlin-swe-bench", "kind": "repo", "url": "https://github.com/Kotlin/kotlin-swe-bench"}, {"kind": "website", "url": "https://kotlinlang.org/benchmark/"}], "categories": ["Code & Software", "Code generation", "Coding Agents", "Kotlin", "Software & AI Compute", "Software engineering"], "description": {"en": "Evaluates AI coding agents on real-world Kotlin tasks from nine open-source repositories. The benchmark uses reproducible Docker environments, regression tests, and a scoring protocol that awards pass only when all expected test transitions are met."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://kotlinlang.org/benchmark/", "source": "claire_radar", "source_url": "https://kotlinlang.org/benchmark/", "title": "Kotlin Benchmark"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:jetbrains-kotlin-benchmark-2026", "modality": null, "name": "Kotlin Benchmark", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-reviewed", "display_eligible": "unknown", "origin_record_id": "bm_kotlin-benchmark_jetbrains-2026", "origin_source": "official-project", "recognition_confidence": "1.0", "record_sha256": "3b3b91f7c1e894787fbcc25637c9c62d51d76bbacb532631a0b6922106752977", "relation": "introduces", "review_state": "source-reviewed", "reviewed_at": "2026-08-19", "source_url": "https://kotlinlang.org/benchmark/"}, "publisher": null, "released": "2026-07-08", "released_reference": {"basis": "first_public", "source_key": "claire-radar:jetbrains-kotlin-benchmark-2026", "source_url": "https://kotlinlang.org/benchmark/"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-jetbrains-kotlin-benchmark-2026", "source": "claire_radar", "source_benchmark_id": "jetbrains-kotlin-benchmark-2026", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": ["Kotlin SWE Benchmark"], "annotation": "Unknown", "applicationDomains": ["Software & AI Compute"], "area": "Code & Software", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/Kotlin/kotlin-swe-bench", "githubScope": "benchmark_repo", "githubStars": 53, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": null, "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"state": "not_applicable"}}, "status": "fresh"}, "capabilities": ["Code generation", "Software engineering"], "confidence": "High", "construction": "Transform Existing", "constructionDetail": "JetBrains adapted the SWE-bench methodology to resolved issues from real Kotlin repositories, packaged with Harbor-compatible containers and regression tests.", "copyGeneration": {"generatedAt": "2026-08-22T18:51:01.308665Z", "inputHash": "12dd9616dd867f1805bf6e8da925809f8216ad775f20f690a97b3c405f36d32d", "model": "deepseek-v4-flash"}, "curation": {"reviewedAt": "2026-08-19", "sources": ["https://blog.jetbrains.com/kotlin/2026/07/introducing-the-kotlin-benchmark-evaluate-ai-coding-agents-on-real-world-kotlin-tasks/", "https://kotlinlang.org/benchmark/methodology/", "https://github.com/Kotlin/kotlin-swe-bench"], "state": "source-reviewed"}, "dataStatus": "primary-source-reviewed", "demo": false, "description": "Evaluates AI coding agents on real-world Kotlin tasks from nine open-source repositories. The benchmark uses reproducible Docker environments, regression tests, and a scoring protocol that awards pass only when all expected test transitions are met.", "domainCuration": {"method": "primary-source-review-v1", "state": "reviewed"}, "evidence": {"reasonCodes": ["official project release", "reviewed non-arXiv source"], "snippet": "The Kotlin Benchmark evaluates AI coding agents on real-world Kotlin tasks using reproducible execution and regression tests."}, "familyId": "bmf_11a0a2a20c95", "firstSeenAt": "2026-08-19", "heat": null, "id": "bm_kotlin-benchmark_jetbrains-2026", "indexedAt": "2026-08-19T00:00:00Z", "industrySectors": ["Software & Cloud"], "links": {"code": "https://github.com/Kotlin/kotlin-swe-bench", "data": null, "paper": null, "pdf": null, "project": "https://kotlinlang.org/benchmark/", "report": "https://blog.jetbrains.com/kotlin/2026/07/introducing-the-kotlin-benchmark-evaluate-ai-coding-agents-on-real-world-kotlin-tasks/"}, "metrics": [{"name": "Tasks", "note": "initial release snapshot", "value": "105"}, {"name": "Repositories", "note": "initial release snapshot", "value": "8"}], "motivation": "Measure whether general coding agents can solve realistic Kotlin repository issues under a reproducible SWE-bench-style protocol.", "name": "Kotlin Benchmark", "oneLine": "Evaluates AI coding agents on real-world Kotlin tasks from nine open-source repositories. The benchmark uses reproducible Docker environments, regression tests, and a scoring protocol that awards pass only when all expected test transitions are met.", "paperTitle": "Introducing the Kotlin Benchmark: Evaluate AI coding agents on real-world Kotlin tasks", "primaryDomain": "Software & AI Compute", "publication": {"evidence": null, "evidenceLevel": "official", "evidenceUrl": "https://blog.jetbrains.com/kotlin/2026/07/introducing-the-kotlin-benchmark-evaluate-ai-coding-agents-on-real-world-kotlin-tasks/", "source": "official-project", "status": "unverified", "venue": null, "verifiedAt": "2026-08-19T00:00:00Z"}, "publications": [], "publishers": [{"name": "JetBrains", "organizationType": "company-research-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/Kotlin/kotlin-swe-bench"}], "ranking": {"90d": {"confidence": "Low", "coverage": 0.55, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.5324576637560364, "value": 53}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": null, "value": null}}, "confidence": "Low", "coverage": 0.55, "rank": 51, "score": 52}, "method": "attention-ranking-v13", "rank": 51, "score": 52}}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releaseDates": {"firstPublicAt": "2026-07-08", "paperV1At": null}, "releasedAt": "2026-07-08", "source": {"authors": ["JetBrains Kotlin"], "categories": ["software-engineering"], "id": "jetbrains-kotlin-benchmark-2026", "title": "Kotlin Benchmark", "type": "official-project", "url": "https://kotlinlang.org/benchmark/"}, "sourceUpdatedAt": "2026-08-19T00:00:00Z", "topics": ["Coding Agents", "Kotlin"], "venueAttempts": [], "whyItMatters": "This benchmark addresses the lack of reference evaluation for Kotlin-specific coding agents, providing a reproducible, task-level framework for comparing agent performance on real-world Kotlin issues and tracking progress."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
