{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2608.16349", "kind": "paper", "url": "https://arxiv.org/abs/2608.16349"}], "authors": ["Yuchen Yuan", "Zhenghuang Wu", "Yuangan Li", "Liang Ma", "Ke Li"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.16349", "source_url": "https://arxiv.org/abs/2608.16349"}, "categories": ["Agents", "Agents & Tool Use", "General AI"], "description": {"en": "AeroCopilotBench is a two-tier benchmark for evaluating LLM agents as aviation copilots. Tier-1 uses 1,200 multiple-choice questions for knowledge assessment, while Tier-2 includes 73 procedural tasks in an interactive virtual cockpit environment, with safety-gated evaluation."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.16349", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.16349", "title": "AeroCopilotBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["Beihang University"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2608.16349", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2608.16349", "source_url": "https://openalex.org/W7203717758"}, "key": "claire-radar:2608.16349", "modality": null, "name": "AeroCopilotBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_aerocopilotbench_7dd468f7", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "cd95431cbb8d23f90a8f55828c1402635e2f2bf1690d556d744556dda11da12a", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2608.16349"}, "publisher": null, "released": "2026-08-17", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.16349", "source_url": "https://arxiv.org/abs/2608.16349"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-16349", "source": "claire_radar", "source_benchmark_id": "2608.16349", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Agents & Tool Use", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2608.16349", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "unavailable"}, "capabilities": [], "confidence": "High", "construction": "Transform Existing", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "1d2a2eb64ae7a62f2646b3d67615636af6a1bdce64fcbaa5a187bb30278a28ce", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "AeroCopilotBench is a two-tier benchmark for evaluating LLM agents as aviation copilots. Tier-1 uses 1,200 multiple-choice questions for knowledge assessment, while Tier-2 includes 73 procedural tasks in an interactive virtual cockpit environment, with safety-gated evaluation.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "This paper presents the AeroCopilot Operational Environment (ACOE), a reproducible interactive virtual-cockpit test environment, and AeroCopilotBench, a two-tier aviation agent evaluation benchmark."}, "familyId": "bmf_cbfa21ffac06", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_aerocopilotbench_7dd468f7", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2608.16349", "paper": "https://arxiv.org/abs/2608.16349", "pdf": "https://arxiv.org/pdf/2608.16349", "project": null, "report": "https://arxiv.org/abs/2608.16349"}, "metrics": [], "motivation": "Large language model (LLM) agents may assist flight crews with complex decisions and task execution, but existing aviation evaluations centered on static knowledge do not support systematic testing of procedural execution and safety compliance in interactive environments.", "name": "AeroCopilotBench", "oneLine": "AeroCopilotBench is a two-tier benchmark for evaluating LLM agents as aviation copilots. Tier-1 uses 1,200 multiple-choice questions for knowledge assessment, while Tier-2 includes 73 procedural tasks in an interactive virtual cockpit environment, with safety-gated evaluation.", "paperTitle": "AeroCopilotBench: A Two-Tier Benchmark for Evaluating LLM Agents as Aviation Copilots in an Interactive Virtual Cockpit Environment", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.16349", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {"90d": {"confidence": "Low", "coverage": 0.0, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": null, "value": null}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "rank": null, "score": null}, "method": "attention-ranking-v13", "rank": null, "score": null}}, "readiness": "Paper only", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-08-17", "source": {"authors": ["Yuchen Yuan", "Zhenghuang Wu", "Yuangan Li", "Liang Ma", "Ke Li"], "categories": ["cs.AI"], "id": "2608.16349", "title": "AeroCopilotBench: A Two-Tier Benchmark for Evaluating LLM Agents as Aviation Copilots in an Interactive Virtual Cockpit Environment", "type": "arxiv", "url": "https://arxiv.org/abs/2608.16349"}, "sourceUpdatedAt": "2026-08-17T09:53:52Z", "topics": ["Agents"], "venueAttempts": [], "whyItMatters": "Aviation evaluations often focus on static knowledge and cannot test procedural execution and safety compliance. AeroCopilotBench provides a reproducible interactive environment and safety-gated scoring to assess agents' task completion and trajectory safety."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
