{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2608.19981", "kind": "paper", "url": "https://arxiv.org/abs/2608.19981"}], "authors": ["Yingjian Chen", "Fan Gao", "Sherry T. Tong", "Haoyu Zhang", "Aosong Feng", "Kevin W. Jin", "Xing Wu", "Jinghui Lu", "Abdul Samad", "Akbar Faruqi", "Cesar Caraballo", "Cibele Brandão", "Dhruva", "Gupta", "Eunji Jeon", "Gabriel Madera-Santiago", "Geon Lee", "Hugo Toshio Itikawa", "Insook Cho", "Isabelli Martins", "Isarar Siddique", "Israr Ahmed", "Jihyo Kwak", "Kanyakorn Veerakanjana", "Luis Guilherme Cardoso", "Minjin Kim", "Piyalitt Ittichaiwong", "Renee Dua", "Santiago Gudiño-Rosales", "Xiujie Chen", "Zeo Lapalus", "Zixin Xu", "Michihiro Yasunaga", "Rex Ying", "Heuiseok Lim", "Jaewoo Kang", "Chanjun Park", "Hang Jiang", "Ethan Goh", "Hyunjae Kim", "Edison Marrese-Taylor", "Yusuke Iwasawa", "Yutaka Matsuo", "Qingyu Chen", "Irene Li"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.19981", "source_url": "https://arxiv.org/abs/2608.19981"}, "categories": ["Biology & Drug Discovery", "Language & Knowledge", "cs.CL"], "description": {"en": "Evaluates multilingual medical LLMs on 1,000 examples per language across nine languages, covering MCQA, NLI, and open-ended QA tasks."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.19981", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.19981", "title": "HealMed"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2608.19981", "modality": null, "name": "HealMed", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "true", "origin_record_id": "bm_healmed_cb6026bf", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "632d160a31a90e52616af48a524e8d83bb8f197d9d3e2c88d4f64710cd5ce4e3", "relation": "introduces", "review_model": "deepseek-v4-pro", "review_state": "ai-reviewed", "reviewed_at": "2026-08-25T23:23:46.758372Z", "source_url": "https://arxiv.org/abs/2608.19981"}, "publisher": null, "released": "2026-08-20", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.19981", "source_url": "https://arxiv.org/abs/2608.19981"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-19981", "source": "claire_radar", "source_benchmark_id": "2608.19981", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["Biology & Drug Discovery"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2608.19981", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "unavailable"}, "attentionForecast": {"confidence": "Medium", "horizon": "7d", "reason": "The benchmark addresses a clear multilingual medical evaluation gap, but the arxiv-only release without explicit code or data links may limit immediate adoption.", "score": 62}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-25T23:23:46.758372Z", "inputHash": "cc5941b7d49014d2f308c1dad339087f544c4bdf316cecab5d88b29b8fffad4f", "model": "deepseek-v4-pro"}, "curation": {"canonicalNameEvidence": "HealMed: Multilingual Evaluation of Large Language Models in Medicine", "canonicalNameSource": "paper_title", "decisionReason": "The paper explicitly presents HealMed as an expert-reviewed benchmark with defined task formats and a public arxiv report, satisfying stable scoring and a public reuse path.", "model": "deepseek-v4-pro", "reviewedAt": "2026-08-25T23:23:46.758372Z", "state": "ai-reviewed"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "Evaluates multilingual medical LLMs on 1,000 examples per language across nine languages, covering MCQA, NLI, and open-ended QA tasks.", "displayEligible": true, "domainCuration": {"method": "rules-v1", "state": "auto"}, "evaluationMode": "score_submission", "evidence": {"reasonCodes": ["exact coined title identity tied to benchmark evidence", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "We present HealMed, an expert-reviewed benchmark for multilingual evaluation of large language models in medicine."}, "familyId": "bmf_a1a70ab23feb", "firstSeenAt": "2026-08-23", "heat": null, "id": "bm_healmed_cb6026bf", "indexedAt": "2026-08-23T06:55:50.392548Z", "industrySectors": ["Pharma & Biotech"], "links": {"code": null, "data": null, "paper": "https://arxiv.org/abs/2608.19981", "pdf": "https://arxiv.org/pdf/2608.19981", "project": null, "report": "https://arxiv.org/abs/2608.19981"}, "metrics": [], "motivation": "We present HealMed, an expert-reviewed benchmark for multilingual evaluation of large language models in medicine.", "name": "HealMed", "oneLine": "Evaluates multilingual medical LLMs on 1,000 examples per language across nine languages, covering MCQA, NLI, and open-ended QA tasks.", "paperTitle": "HealMed: Multilingual Evaluation of Large Language Models in Medicine", "primaryDomain": "Biology & Drug Discovery", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.19981", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-08-23T06:55:50.392548Z"}, "publications": [], "publishers": [{"name": "Medical Experts and Physicians Across Nine Countries", "organizationType": "academic-lab", "role": "benchmark-publisher", "sourceUrl": "https://arxiv.org/abs/2608.19981"}], "ranking": {"90d": {"confidence": "Low", "coverage": 0.0, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": null, "value": null}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "rank": null, "score": null}, "method": "attention-ranking-v13", "rank": null, "score": null}}, "readiness": "Paper only", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-08-20", "source": {"authors": ["Yingjian Chen", "Fan Gao", "Sherry T. Tong", "Haoyu Zhang", "Aosong Feng", "Kevin W. Jin", "Xing Wu", "Jinghui Lu", "Abdul Samad", "Akbar Faruqi", "Cesar Caraballo", "Cibele Brand\\~ao", "Dhruva (Drew) Gupta", "Eunji Jeon", "Gabriel Madera-Santiago", "Geon Lee", "Hugo Toshio Itikawa", "Insook Cho", "Isabelli Martins", "Isarar Siddique", "Israr Ahmed", "Jihyo Kwak", "Kanyakorn Veerakanjana", "Luis Guilherme Cardoso", "Minjin Kim", "Piyalitt Ittichaiwong", "Renee Dua", "Santiago Gudi\\~no-Rosales", "Xiujie Chen", "Zeo Lapalus", "Zixin Xu", "Michihiro Yasunaga", "Rex Ying", "Heuiseok Lim", "Jaewoo Kang", "Chanjun Park", "Hang Jiang", "Ethan Goh", "Hyunjae Kim", "Edison Marrese-Taylor", "Yusuke Iwasawa", "Yutaka Matsuo", "Qingyu Chen and Irene Li"], "categories": ["cs.CL"], "id": "2608.19981", "title": "HealMed: Multilingual Evaluation of Large Language Models in Medicine", "type": "arxiv", "url": "https://arxiv.org/abs/2608.19981"}, "sourceUpdatedAt": "2026-08-20T12:52:09Z", "topics": ["cs.CL"], "venueAttempts": [], "whyItMatters": "Fills a gap in multilingual medical evaluation by measuring performance across language resource levels and assessing whether translation quality affects cross-language results."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
