{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2606.25402", "kind": "paper", "url": "https://arxiv.org/abs/2606.25402"}], "authors": ["Daniele Cipollone", "Sergey Titov", "Maliheh Izadi", "Egor Bogomolov", "Arie van Deursen"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2606.25402", "source_url": "https://arxiv.org/abs/2606.25402"}, "categories": ["Code", "Code & Software", "Code generation", "Factuality", "Software & AI Compute"], "description": {"en": "LibEvoBench is a multi-task benchmark for evaluating code generation models on API evolution across versions of Python libraries. It includes tasks spanning multiple library versions and introduces the Software Evolution Understanding Score (SEUS) to measure version-specific knowledge consistency."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2606.25402", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2606.25402", "title": "LibEvoBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["Delft University of Technology"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2606.25402", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2606.25402", "source_url": "https://openalex.org/W7165943973"}, "key": "claire-radar:2606.25402", "modality": null, "name": "LibEvoBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_libevobench_4c11745f", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "1ae549bcf2ded179a5b877c26246e369e31773f753e5093ca8d5509a48bfd1ca", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2606.25402"}, "publisher": null, "released": "2026-06-24", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2606.25402", "source_url": "https://arxiv.org/abs/2606.25402"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2606-25402", "source": "claire_radar", "source_benchmark_id": "2606.25402", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["Software & AI Compute"], "area": "Code & Software", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2606.25402", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "unavailable"}, "capabilities": ["Code generation", "Factuality"], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "7ae1075acff0c0c02977d3297cabbb264cd8d1fa8a08ba414458daa3221d339d", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "LibEvoBench is a multi-task benchmark for evaluating code generation models on API evolution across versions of Python libraries. It includes tasks spanning multiple library versions and introduces the Software Evolution Understanding Score (SEUS) to measure version-specific knowledge consistency.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "To systematically evaluate this phenomenon, we introduce LibEvoBench, a multi-task benchmark spanning multiple versions of widely used Python libraries, along with a new metric, the Software Evolution Understanding Score (SEUS), to measure models' consistency when working with evolving APIs."}, "familyId": "bmf_0e8e47aebe59", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_libevobench_4c11745f", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": ["Software & Cloud"], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2606.25402", "paper": "https://arxiv.org/abs/2606.25402", "pdf": "https://arxiv.org/pdf/2606.25402", "project": null, "report": "https://arxiv.org/abs/2606.25402"}, "metrics": [], "motivation": "Large software projects often depend on older versions of libraries, even as APIs continue to evolve across releases.", "name": "LibEvoBench", "oneLine": "LibEvoBench is a multi-task benchmark for evaluating code generation models on API evolution across versions of Python libraries. It includes tasks spanning multiple library versions and introduces the Software Evolution Understanding Score (SEUS) to measure version-specific knowledge consistency.", "paperTitle": "LibEvoBench: Probing Temporal Knowledge Stratification in Code Generation Models", "primaryDomain": "Software & AI Compute", "publication": {"evidence": "Accepted at the DL4Code workshop at ICML 2026", "evidenceLevel": "author-claim", "evidenceUrl": "https://arxiv.org/abs/2606.25402", "source": "arxiv-comments", "status": "acceptance_claimed", "venue": "DL4Code workshop at ICML 2026", "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "arXiv", "organizationType": "community", "role": "benchmark-publisher", "sourceUrl": "https://arxiv.org/abs/2606.25402"}], "ranking": {}, "readiness": "Paper only", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-06-24", "source": {"authors": ["Daniele Cipollone", "Sergey Titov", "Maliheh Izadi", "Egor Bogomolov", "Arie van Deursen"], "categories": ["cs.SE", "cs.AI"], "id": "2606.25402", "title": "LibEvoBench: Probing Temporal Knowledge Stratification in Code Generation Models", "type": "arxiv", "url": "https://arxiv.org/abs/2606.25402"}, "sourceUpdatedAt": "2026-06-24T04:58:28Z", "topics": ["Code"], "venueAttempts": [{"decisionRaw": "Accepted at the DL4Code workshop at ICML 2026", "evidence": [{"level": "author-claim", "observedAt": "2026-10-14T01:46:58.058199Z", "rawValue": "Accepted at the DL4Code workshop at ICML 2026", "sourceType": "arxiv-comments", "sourceUrl": "https://arxiv.org/abs/2606.25402"}], "reviewStatus": "accepted", "venueName": "DL4Code workshop at ICML 2026"}], "whyItMatters": "Addresses the gap in evaluating models on version-specific API knowledge, which is critical for real-world software maintenance. It highlights limitations of current training on temporally mixed corpora and motivates temporally grounded learning."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
