{"record": {"aliases": ["$\\Phi$-Bench: Can Large Language Models Engineer the Infrastructure That Powers Them?"], "artifacts": [{"id": "arxiv:2609.10226", "kind": "paper", "url": "https://arxiv.org/abs/2609.10226"}], "authors": ["Leilei Ding", "Shumin Wang", "Yuting Huang", "Fanqi Wan", "Yinmin Zhang", "Qi Han", "Yiming Xu", "Feiyuan Zhang", "Xiaomeng Chu", "Guoliang You", "Wuyang Zhang", "Daxin Jiang", "Yanyong Zhang"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2609.10226", "source_url": "https://arxiv.org/abs/2609.10226"}, "categories": ["Language & Knowledge", "Software & AI Compute", "cs.CL"], "description": {"en": "Evaluates LLMs on tasks across the LLM infrastructure stack, from kernel completion to end-to-end system optimization using real-world code repositories."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2609.10226", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2609.10226", "title": "Φ-Bench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["University of Science and Technology of China", "Peking University", "Hong Kong University of Science and Technology", "Yale University", "University of Pennsylvania"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2609.10226", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2609.10226", "source_url": "https://openalex.org/W7212141264"}, "key": "claire-radar:2609.10226", "modality": null, "name": "Φ-Bench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"admission_policy_version": "2026-09-09.1", "confidence": "Medium", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "true", "origin_record_id": "bm_phi-bench-can-large-language-models-engine_4e5601b7", "origin_source": "arxiv", "recognition_confidence": "0.6", "record_sha256": "fa9102d43a22134b32afaf8c344527ccb1fce9447623e3fed4568b0dbc0163be", "relation": "introduces", "review_model": "deepseek-v4-pro", "review_state": "ai-reviewed", "reviewed_at": "2026-09-11T00:52:53.785364Z", "source_url": "https://arxiv.org/abs/2609.10226"}, "publisher": null, "released": "2026-09-09", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2609.10226", "source_url": "https://arxiv.org/abs/2609.10226"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2609-10226", "source": "claire_radar", "source_benchmark_id": "2609.10226", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": ["$\\Phi$-Bench: Can Large Language Models Engineer the Infrastructure That Powers Them?"], "annotation": "Unknown", "applicationDomains": ["Software & AI Compute"], "area": "Language & Knowledge", "attention": {"asOf": "2026-09-10", "attemptedAt": "2026-09-10T12:24:43.080049Z", "githubRepo": "https://github.com/one2piece2hello/faibench_Frontier_InfraBench", "githubScope": "benchmark_repo", "githubStars": 54, "hfDailySubmittedAt": "2026-09-10T00:00:00.000Z", "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 5, "hfPaperUrl": "https://huggingface.co/papers/2609.10226", "observedAt": "2026-09-10T12:24:43.080049Z", "signalStatus": {"githubStars": {"observedAt": "2026-09-10T12:24:43.080049Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"observedAt": "2026-09-10T12:24:43.080049Z", "state": "fresh"}}, "status": "fresh"}, "attentionForecast": {"confidence": "Medium", "horizon": "7d", "reason": "LLM infrastructure engineering is an emerging area with high interest from systems and AI research communities.", "score": 60}, "capabilities": [], "confidence": "Medium", "construction": "Transform Existing", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-09-11T00:52:53.785364Z", "inputHash": "00f0b639ef0ac8c43aacec890e6ce9a61c3542f8301acf68cb793e327e03585f", "model": "deepseek-v4-pro"}, "curation": {"admissionPolicyVersion": "2026-09-09.1", "canonicalNameEvidence": "we present $Φ$-Bench, a benchmark for systematically evaluating LLMs on engineering the LLM infrastructure stack", "canonicalNameSource": "abstract", "decisionReason": "Clear benchmark identity and task diversity, though no direct code/data link; potential for reuse from described grounding.", "evaluationEvidence": {"reuse": "Derived from optimization problems studied in frontier research and grounded in real-world code repositories", "scoring": "Extensive experiments on frontier LLMs reveal their current capabilities and limitations in engineering complex LLM infrastructure", "task": "evaluating LLMs on engineering the LLM infrastructure stack; spans tasks of varying complexity, ranging from localized kernel-level function completion to long-horizon implementation and end-to-end system optimization."}, "model": "deepseek-v4-pro", "reviewedAt": "2026-09-11T00:52:53.785364Z", "state": "ai-reviewed"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "Evaluates LLMs on tasks across the LLM infrastructure stack, from kernel completion to end-to-end system optimization using real-world code repositories.", "displayEligible": true, "domainCuration": {"method": "rules-v1", "state": "auto"}, "evaluationMode": "public_reusable", "evidence": {"reasonCodes": ["benchmark term in abstract", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "To address this gap, we present $\\Phi$-Bench, a benchmark for systematically evaluating LLMs on engineering the LLM infrastructure stack."}, "familyId": "bmf_720c90680ead", "firstSeenAt": "2026-09-10", "heat": null, "id": "bm_phi-bench-can-large-language-models-engine_4e5601b7", "indexedAt": "2026-09-10T06:16:58.931316Z", "industrySectors": ["Software & Cloud"], "links": {"code": null, "data": null, "paper": "https://arxiv.org/abs/2609.10226", "pdf": "https://arxiv.org/pdf/2609.10226", "project": null, "report": "https://arxiv.org/abs/2609.10226"}, "metrics": [], "motivation": "Large language models (LLMs) have demonstrated remarkable capabilities in reasoning and code generation, raising the prospect that they could assist in developing and optimizing the very infrastructure that powers them.", "name": "Φ-Bench", "oneLine": "Evaluates LLMs on tasks across the LLM infrastructure stack, from kernel completion to end-to-end system optimization using real-world code repositories.", "paperTitle": "$\\Phi$-Bench: Can Large Language Models Engineer the Infrastructure That Powers Them?", "primaryDomain": "Software & AI Compute", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2609.10226", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-09-10T06:16:58.931316Z"}, "publications": [], "ranking": {"30d": {"confidence": "High", "coverage": 0.85, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 30}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.5968470963585285, "value": 54}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.31738133432377996, "value": 5}}, "confidence": "High", "coverage": 0.85, "rank": 30, "score": 50}, "method": "attention-ranking-v13", "rank": 30, "score": 50}, "90d": {"confidence": "Medium", "coverage": 0.7, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.5408637316324297, "value": 54}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.31738133432377996, "value": 5}}, "confidence": "Medium", "coverage": 0.7, "rank": 62, "score": 50}, "method": "attention-ranking-v13", "rank": 62, "score": 50}, "today": {"confidence": "Medium", "coverage": 0.7, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": 1, "historyDate": "2026-09-09", "rank": null, "score": null, "windowDays": 1}, "level": {"components": {"githubStars": {"normalization": "percentile", "normalized": 0.9166666666666666, "percentile": 0.9166666666666666, "value": 54}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": 0.8333333333333334, "percentile": 0.8333333333333334, "value": 5}, "llmAttentionForecast": {"maxContribution": 0.25, "percentile": 0.3, "role": "bonus", "value": 30}}, "confidence": "Medium", "coverage": 0.7, "rank": 1, "score": 70}, "method": "attention-ranking-v13", "rank": 1, "score": 70}}, "readiness": "Paper only", "recognitionConfidence": 0.6, "relation": "introduces", "releasedAt": "2026-09-09", "source": {"authors": ["Leilei Ding", "Shumin Wang", "Yuting Huang", "Fanqi Wan", "Yinmin Zhang", "Qi Han", "Yiming Xu", "Feiyuan Zhang", "Xiaomeng Chu", "Guoliang You", "Wuyang Zhang", "Daxin Jiang", "Yanyong Zhang"], "categories": ["cs.CL"], "id": "2609.10226", "title": "$\\Phi$-Bench: Can Large Language Models Engineer the Infrastructure That Powers Them?", "type": "arxiv", "url": "https://arxiv.org/abs/2609.10226"}, "sourceUpdatedAt": "2026-09-09T14:23:47Z", "topics": ["cs.CL"], "venueAttempts": [], "whyItMatters": "Addresses the gap in open-ended, long-horizon infrastructure engineering evaluation, informing progress toward autonomous optimization of AI systems."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
