{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2609.10016", "kind": "paper", "url": "https://arxiv.org/abs/2609.10016"}, {"id": "gh:continker/metrollm-bench", "kind": "repo", "url": "https://github.com/continker/metrollm-bench"}], "authors": ["Remco Hendriks"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2609.10016", "source_url": "https://arxiv.org/abs/2609.10016"}, "categories": ["General AI", "Language & Knowledge", "cs.LG"], "description": {"en": "MetroLLM-Bench evaluates language models as transit kiosk policy layers across six real metro systems. It includes 955 cases covering routing, fare calculation, disruptions, accessibility, and adversarial input, with models required to call structured tools and submit machine-renderable terminal states."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2609.10016", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2609.10016", "title": "MetroLLM-Bench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2609.10016", "modality": null, "name": "MetroLLM-Bench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"admission_policy_version": "2026-09-09.1", "confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "true", "origin_record_id": "bm_metrollm-bench_4cec77cf", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "d4cb57cbf61b51755a77b14d96f0bccf11b97a27bebe146c23080006b8d00363", "relation": "introduces", "review_model": "deepseek-v4-pro", "review_state": "ai-reviewed", "reviewed_at": "2026-09-11T00:52:53.785364Z", "source_url": "https://arxiv.org/abs/2609.10016"}, "publisher": null, "released": "2026-09-09", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2609.10016", "source_url": "https://arxiv.org/abs/2609.10016"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2609-10016", "source": "claire_radar", "source_benchmark_id": "2609.10016", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-09-10", "attemptedAt": "2026-09-10T12:24:43.080049Z", "githubRepo": "https://github.com/continker/metrollm-bench", "githubScope": "benchmark_repo", "githubStars": 0, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 0, "hfPaperUrl": "https://huggingface.co/papers/2609.10016", "observedAt": "2026-09-10T12:24:43.080049Z", "signalStatus": {"githubStars": {"observedAt": "2026-09-10T12:24:43.080049Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"observedAt": "2026-09-10T12:24:43.080049Z", "state": "fresh"}}, "status": "fresh"}, "attentionForecast": {"confidence": "Medium", "horizon": "7d", "reason": "The benchmark targets a concrete, broadly understandable real-world application with released code, models, and a live demo, likely generating interest from both applied ML practitioners and transit system developers.", "score": 75}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-09-11T00:52:53.785364Z", "inputHash": "12e8310744bb916b2e9114627073162627aeb27ab63ef5a1698a29c670995c5d", "model": "deepseek-v4-pro"}, "curation": {"admissionPolicyVersion": "2026-09-09.1", "canonicalNameEvidence": "MetroLLM-Bench: Evaluating Language Models as Transit Kiosk Runtimes", "canonicalNameSource": "paper_title", "decisionReason": "The benchmark is formally named in the paper title, includes detailed task definitions, deterministic Tier 1 and LLM-judged Tier 2 scoring, and a public GitHub release with harness, reproduction guide, and models.", "evaluationEvidence": {"reuse": "The benchmark, harness, reproduction guide, and fine-tuned students are released at https://github.com/continker/metrollm-bench.", "scoring": "Fourteen deterministic scoring components form Tier 1; eight semantic-quality components form Tier 2, six of which use a language-model judge. We report Tier 1 and the combined score of both tiers.", "task": "It is a 955-case benchmark across six metro systems. Each case asks whether a language model can run a transit kiosk from a prose prompt, handling routing, fares, and disruptions through tool calls, with no code changes."}, "model": "deepseek-v4-pro", "reviewedAt": "2026-09-11T00:52:53.785364Z", "state": "ai-reviewed"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "MetroLLM-Bench evaluates language models as transit kiosk policy layers across six real metro systems. It includes 955 cases covering routing, fare calculation, disruptions, accessibility, and adversarial input, with models required to call structured tools and submit machine-renderable terminal states.", "displayEligible": true, "domainCuration": {"method": "rules-v1", "state": "auto"}, "evaluationMode": "score_submission", "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "We introduce MetroLLM-Bench, a 955-case benchmark for testing language models as the policy layer of a transit kiosk."}, "familyId": "bmf_9c1d0c89acb4", "firstSeenAt": "2026-09-10", "heat": null, "id": "bm_metrollm-bench_4cec77cf", "indexedAt": "2026-09-10T00:47:26.889471Z", "industrySectors": [], "links": {"code": "https://github.com/continker/metrollm-bench", "data": null, "paper": "https://arxiv.org/abs/2609.10016", "pdf": "https://arxiv.org/pdf/2609.10016", "project": null, "report": "https://arxiv.org/abs/2609.10016"}, "metrics": [], "motivation": "We introduce MetroLLM-Bench, a 955-case benchmark for testing language models as the policy layer of a transit kiosk.", "name": "MetroLLM-Bench", "oneLine": "MetroLLM-Bench evaluates language models as transit kiosk policy layers across six real metro systems. It includes 955 cases covering routing, fare calculation, disruptions, accessibility, and adversarial input, with models required to call structured tools and submit machine-renderable terminal states.", "paperTitle": "MetroLLM-Bench: Evaluating Language Models as Transit Kiosk Runtimes", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2609.10016", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-09-10T00:47:26.889471Z"}, "publications": [], "ranking": {"30d": {"confidence": "High", "coverage": 0.85, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 30}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.0, "value": 0}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.0, "value": 0}}, "confidence": "High", "coverage": 0.85, "rank": 240, "score": 8}, "method": "attention-ranking-v13", "rank": 240, "score": 8}, "90d": {"confidence": "Medium", "coverage": 0.7, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.0, "value": 0}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.0, "value": 0}}, "confidence": "Medium", "coverage": 0.7, "rank": 458, "score": 15}, "method": "attention-ranking-v13", "rank": 458, "score": 15}, "today": {"confidence": "Medium", "coverage": 0.7, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": 1, "historyDate": "2026-09-09", "rank": null, "score": null, "windowDays": 1}, "level": {"components": {"githubStars": {"normalization": "percentile", "normalized": 0.25, "percentile": 0.25, "value": 0}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": 0.16666666666666666, "percentile": 0.16666666666666666, "value": 0}, "llmAttentionForecast": {"maxContribution": 0.25, "percentile": 0.62, "role": "bonus", "value": 62}}, "confidence": "Medium", "coverage": 0.7, "rank": 6, "score": 32}, "method": "attention-ranking-v13", "rank": 6, "score": 32}}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-09-09", "source": {"authors": ["Remco Hendriks (Continker)"], "categories": ["cs.LG", "cs.AI", "cs.CL"], "id": "2609.10016", "title": "MetroLLM-Bench: Evaluating Language Models as Transit Kiosk Runtimes", "type": "arxiv", "url": "https://arxiv.org/abs/2609.10016"}, "sourceUpdatedAt": "2026-09-09T10:46:56Z", "topics": ["cs.LG"], "venueAttempts": [], "whyItMatters": "It provides a reproducible, deterministic protocol for testing offline language models in public transit kiosks, addressing sovereignty and practicality for edge deployment where small fine-tuned models may replace cloud APIs."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
