{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2606.03657", "kind": "paper", "url": "https://arxiv.org/abs/2606.03657"}], "authors": ["Jinnuo Liu", "Yue Peng", "Jinhan Niu", "Hongyi Wen"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2606.03657", "source_url": "https://arxiv.org/abs/2606.03657"}, "categories": ["Agents", "Agents & Tool Use", "Factuality", "Software & AI Compute", "Tool use"], "description": {"en": "NovelAPIBench is a dynamic benchmark for evaluating LLM tool use with novel APIs, covering knowledge components and diagnostic failure categories."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2606.03657", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2606.03657", "title": "NovelAPIBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["New York University Shanghai"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2606.03657", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2606.03657", "source_url": "https://openalex.org/W7163425912"}, "key": "claire-radar:2606.03657", "modality": null, "name": "NovelAPIBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_novelapibench_5fb13ed7", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "f70558359c8142241d7aa6543f28f8f710f89abf3bb3bc2244ecdf13a9ac50d0", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2606.03657"}, "publisher": null, "released": "2026-06-02", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2606.03657", "source_url": "https://arxiv.org/abs/2606.03657"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2606-03657", "source": "claire_radar", "source_benchmark_id": "2606.03657", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Machine Generated", "applicationDomains": ["Software & AI Compute"], "area": "Agents & Tool Use", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 1, "hfPaperUrl": "https://huggingface.co/papers/2606.03657", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-25", "state": "stale"}}, "status": "stale"}, "capabilities": ["Tool use", "Factuality"], "confidence": "High", "construction": "Original Synthetic", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "36764ea5906874a108e686ec7760c9b7b3aa6bb5576dfabdf0c32a5e81bec8d1", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "NovelAPIBench is a dynamic benchmark for evaluating LLM tool use with novel APIs, covering knowledge components and diagnostic failure categories.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["exact named benchmark artifact released in abstract", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "We introduce NovelAPIBench, a fully automated dynamic benchmark that, for any base model and target library, discovers novel APIs, extracts decomposed knowledge bundles, generates executable coding tasks, and assigns failed samples to six diagnostic categories."}, "familyId": "bmf_05bbd7d1ac95", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_novelapibench_5fb13ed7", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": ["Software & Cloud"], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2606.03657", "paper": "https://arxiv.org/abs/2606.03657", "pdf": "https://arxiv.org/pdf/2606.03657", "project": null, "report": "https://arxiv.org/abs/2606.03657"}, "metrics": [], "motivation": "Large language models for code generation often need to use APIs that are absent from their pretraining data.", "name": "NovelAPIBench", "oneLine": "NovelAPIBench is a dynamic benchmark for evaluating LLM tool use with novel APIs, covering knowledge components and diagnostic failure categories.", "paperTitle": "Diagnosing Knowledge Gaps in LLM Tool Use: An Agentic Benchmark for Novel API Acquisition", "primaryDomain": "Software & AI Compute", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2606.03657", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {}, "readiness": "Paper only", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-06-02", "source": {"authors": ["Jinnuo Liu", "Yue Peng", "Jinhan Niu", "Hongyi Wen"], "categories": ["cs.AI"], "id": "2606.03657", "title": "Diagnosing Knowledge Gaps in LLM Tool Use: An Agentic Benchmark for Novel API Acquisition", "type": "arxiv", "url": "https://arxiv.org/abs/2606.03657"}, "sourceUpdatedAt": "2026-06-02T13:46:04Z", "topics": ["Agents"], "venueAttempts": [], "whyItMatters": "It addresses the gap in evaluating models' ability to acquire new APIs, providing insights into the complementary roles of retrieval and fine-tuning."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
