{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2608.10692", "kind": "paper", "url": "https://arxiv.org/abs/2608.10692"}, {"id": "hf:junjie-ye/spieval", "kind": "dataset", "url": "https://huggingface.co/datasets/Junjie-Ye/SPIEval"}], "authors": ["Junjie Ye", "Zhuohui Sheng", "Shaofan Liu", "Yulun Zhu", "Wenjie Fu", "Dingwei Zhu", "Ming Zhang", "Yujiong Shen", "Weichao Wang", "Xin Zhao", "Shihan Dou", "Tao Gui", "Qi Zhang", "Xuanjing Huang", "Pluto Zhou"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.10692", "source_url": "https://arxiv.org/abs/2608.10692"}, "categories": ["General AI", "Language & Knowledge", "Reasoning"], "description": {"en": "SPIEval is a human-curated benchmark for evaluating large language models as mobile assistants that retrieve and reason over personal information scattered across multiple apps. It comprises 250 tasks across five cognitive capabilities, 4,335 fictional personal records in 10 simulated apps, and supports multi-turn interaction through 21 tools. Models receive underspecified user instructions and must search records and invoke tools; final execution calls are compared to human-annotated gold calls at the parameter level."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.10692", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.10692", "title": "SPIEval"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["Tencent (China)", "Fudan University"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2608.10692", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2608.10692", "source_url": "https://openalex.org/W7202281988"}, "key": "claire-radar:2608.10692", "modality": null, "name": "SPIEval", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_spieval_a21bc309", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "156310473d8962a1ebb3504d7d4ba69e7c5dfa7507d2aaf6b15d505d03883c29", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2608.10692"}, "publisher": null, "released": "2026-08-11", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.10692", "source_url": "https://arxiv.org/abs/2608.10692"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-10692", "source": "claire_radar", "source_benchmark_id": "2608.10692", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": "2026-08-12T00:00:00.000Z", "hfDataset": "Junjie-Ye/SPIEval", "hfDatasetDownloads": 95, "hfDatasetLikes": 1, "hfPaperUpvotes": 12, "hfPaperUrl": "https://huggingface.co/papers/2608.10692", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetLikes": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfPaperUpvotes": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}}, "status": "fresh"}, "capabilities": ["Reasoning"], "confidence": "High", "construction": "Transform Existing", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "edf6c3676d14db9361b58c1f98994c02d946f5484df4db8cf02829bc5ba991c2", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "SPIEval is a human-curated benchmark for evaluating large language models as mobile assistants that retrieve and reason over personal information scattered across multiple apps. It comprises 250 tasks across five cognitive capabilities, 4,335 fictional personal records in 10 simulated apps, and supports multi-turn interaction through 21 tools. Models receive underspecified user instructions and must search records and invoke tools; final execution calls are compared to human-annotated gold calls at the parameter level.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["exact coined title identity tied to benchmark evidence", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "To address this gap, we introduce SPIEval, a human-curated benchmark grounded in five cognitive capabilities (i.e., reasoning, disambiguation, integration, preference inference, and multi-intent decomposition)."}, "familyId": "bmf_8905e75a39cb", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_spieval_a21bc309", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": null, "data": "https://huggingface.co/datasets/Junjie-Ye/SPIEval", "hfPaper": "https://huggingface.co/papers/2608.10692", "paper": "https://arxiv.org/abs/2608.10692", "pdf": "https://arxiv.org/pdf/2608.10692", "project": null, "report": "https://arxiv.org/abs/2608.10692"}, "metrics": [], "motivation": "Large language models (LLMs) are increasingly deployed as mobile assistants, where a key challenge is leveraging personal information scattered across multiple applications (apps) to complete user instructions.", "name": "SPIEval", "oneLine": "SPIEval is a human-curated benchmark for evaluating large language models as mobile assistants that retrieve and reason over personal information scattered across multiple apps. It comprises 250 tasks across five cognitive capabilities, 4,335 fictional personal records in 10 simulated apps, and supports multi-turn interaction through 21 tools. Models receive underspecified user instructions and must search records and invoke tools; final execution calls are compared to human-annotated gold calls at the parameter level.", "paperTitle": "SPIEval: Evaluating Large Language Models as Mobile Assistants over Scattered Personal Information", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.10692", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {"90d": {"confidence": "Medium", "coverage": 0.45, "datasetDownloadRank": 61, "datasetRankPopulation": 85, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": null, "value": null}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": 0.4700362247705979, "value": 95}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.4532127250650379, "value": 12}}, "confidence": "Medium", "coverage": 0.45, "rank": 78, "score": 48}, "method": "attention-ranking-v13", "rank": 78, "score": 48}}, "readiness": "Inspectable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-08-11", "source": {"authors": ["Junjie Ye", "Zhuohui Sheng", "Shaofan Liu", "Yulun Zhu", "Wenjie Fu", "Dingwei Zhu", "Ming Zhang", "Yujiong Shen", "Weichao Wang", "Xin Zhao", "Shihan Dou", "Tao Gui", "Qi Zhang", "Xuanjing Huang", "Pluto Zhou"], "categories": ["cs.CL", "cs.AI"], "id": "2608.10692", "title": "SPIEval: Evaluating Large Language Models as Mobile Assistants over Scattered Personal Information", "type": "arxiv", "url": "https://arxiv.org/abs/2608.10692"}, "sourceUpdatedAt": "2026-08-11T09:14:53Z", "topics": ["Reasoning"], "venueAttempts": [], "whyItMatters": "Existing benchmarks do not target the challenge of leveraging scattered personal information across apps in mobile assistant settings. SPIEval provides a controlled, verifiable evaluation environment grounded in five cognitive capabilities, enabling assessment of model capabilities in realistic mobile contexts. The reported results show substantial room for improvement and highlight fundamental limitations in information localization and search efficiency, offering practical guidance for deploying LLM-based assistants."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
