{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2608.16022", "kind": "paper", "url": "https://arxiv.org/abs/2608.16022"}, {"kind": "website", "url": "https://bench.matrix.openharmony.cn/"}], "authors": ["Li Li", "Han Hu", "Tianjian Zhang", "Xin Peng", "Fangzhu Mao", "Qingyu Zhang", "Xiaoheng Xie", "Zhongmin Tang", "Zhihao Lin", "Haolin Ruan", "Miaomiao Dong", "Liuchuan Zhu", "Yue Li", "Chi Chen", "Wenkang Zhong", "Mingfei Zhang", "Yang Yu", "Bo Sun", "Chaorui Zhang", "Weixi Zhang", "Wei Han", "Bo Bai", "Kui Liu", "Gang Fan", "Siru Liu", "Jiaqian Zhou", "Jiali Sun", "Yunbiao Dong", "Wenhao Zhong", "Yunhong Xu"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.16022", "source_url": "https://arxiv.org/abs/2608.16022"}, "categories": ["Code & Software", "General AI", "cs.SE"], "description": {"en": "OpenHarmony Bench evaluates coding agents on 153 app-level ArkTS tasks across three input sources, with 242 feature points and device-based verification."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.16022", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.16022", "title": "OpenHarmony Bench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2608.16022", "modality": null, "name": "OpenHarmony Bench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_openharmony-bench_a2c42703", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "f26766ec5c2080d32460a12d7b32eef79c8116caeff7237a2556b6f8f0a9a3a5", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2608.16022"}, "publisher": null, "released": "2026-08-17", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.16022", "source_url": "https://arxiv.org/abs/2608.16022"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-16022", "source": "claire_radar", "source_benchmark_id": "2608.16022", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Code & Software", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 3, "hfPaperUrl": "https://huggingface.co/papers/2608.16022", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-30", "state": "stale"}}, "status": "stale"}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "a954e61b07650b8569ccbd8e7f1dc95b0b4603681721a0ca94040c81faf4537a", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "OpenHarmony Bench evaluates coding agents on 153 app-level ArkTS tasks across three input sources, with 242 feature points and device-based verification.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "We present OPENHARMONY BENCH, an app-level coding benchmark for evaluating LLM-based coding agents on OpenHarmony ArkTS applications."}, "familyId": "bmf_ad07fa1e0a63", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_openharmony-bench_a2c42703", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2608.16022", "paper": "https://arxiv.org/abs/2608.16022", "pdf": "https://arxiv.org/pdf/2608.16022", "project": "https://bench.matrix.openharmony.cn/", "report": "https://arxiv.org/abs/2608.16022"}, "metrics": [], "motivation": "We present OPENHARMONY BENCH, an app-level coding benchmark for evaluating LLM-based coding agents on OpenHarmony ArkTS applications.", "name": "OpenHarmony Bench", "oneLine": "OpenHarmony Bench evaluates coding agents on 153 app-level ArkTS tasks across three input sources, with 242 feature points and device-based verification.", "paperTitle": "OpenHarmony Bench: Evaluating LLMs and Coding Agents on OpenHarmony App Development", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.16022", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "OpenHarmony Community", "organizationType": "community", "role": "benchmark-publisher", "sourceUrl": "https://bench.matrix.openharmony.cn/"}], "ranking": {"90d": {"confidence": "Low", "coverage": 0.15, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": null, "value": null}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.24495074076910422, "value": 3}}, "confidence": "Low", "coverage": 0.15, "rank": null, "score": 46}, "method": "attention-ranking-v13", "rank": null, "score": 46}}, "readiness": "Inspectable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-08-17", "source": {"authors": ["Li Li", "Han Hu", "Tianjian Zhang", "Xin Peng", "Fangzhu Mao", "Qingyu Zhang", "Xiaoheng Xie", "Zhongmin Tang", "Zhihao Lin", "Haolin Ruan", "Miaomiao Dong", "Liuchuan Zhu", "Yue Li", "Chi Chen", "Wenkang Zhong", "Mingfei Zhang", "Yang Yu", "Bo Sun", "Chaorui Zhang", "Weixi Zhang", "Wei Han", "Bo Bai", "Kui Liu", "Gang Fan", "Siru Liu", "Jiaqian Zhou", "Jiali Sun", "Yunbiao Dong", "Wenhao Zhong", "Yunhong Xu"], "categories": ["cs.SE"], "id": "2608.16022", "title": "OpenHarmony Bench: Evaluating LLMs and Coding Agents on OpenHarmony App Development", "type": "arxiv", "url": "https://arxiv.org/abs/2608.16022"}, "sourceUpdatedAt": "2026-08-17T02:27:10Z", "topics": ["cs.SE"], "venueAttempts": [], "whyItMatters": "It measures end-to-end app-level correctness, filling the gap between function-level coding benchmarks and real-world app development."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
