{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2607.23123", "kind": "paper", "url": "https://arxiv.org/abs/2607.23123"}, {"id": "gh:shaqiu-ai/sqbench", "kind": "repo", "url": "https://github.com/shaqiu-ai/SQBench"}], "authors": ["Summer Sun"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2607.23123", "source_url": "https://arxiv.org/abs/2607.23123"}, "categories": ["General AI", "Language & Knowledge", "cs.AI"], "description": {"en": "SQBench v1.0 evaluates language-model agents on 220 production-oriented tasks organized into L1 atomic capabilities, L2 composite skills, and L3 business scenarios. Tasks require processing input assets, using tools, and producing a specified deliverable. Scoring computes Completion, Risk Penalty, and Performance from a 10D Risk Matrix; Strict Pass requires Completion=1 and Risk Penalty=0."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2607.23123", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2607.23123", "title": "SQBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2607.23123", "modality": null, "name": "SQBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_sqbench_9c1d0337", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "5e7d767f8ab8e117ec4961d25d58be44c21efe2554c809222ccd09c340c92815", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2607.23123"}, "publisher": null, "released": "2026-07-25", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2607.23123", "source_url": "https://arxiv.org/abs/2607.23123"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2607-23123", "source": "claire_radar", "source_benchmark_id": "2607.23123", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/shaqiu-ai/SQBench", "githubScope": "benchmark_repo", "githubStars": 0, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2607.23123", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "fresh"}, "capabilities": [], "confidence": "High", "construction": "Unknown", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "0a99db05505fe6f210c3dcaf8248dbaff264e6b7172899589d277490580ac512", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "SQBench v1.0 evaluates language-model agents on 220 production-oriented tasks organized into L1 atomic capabilities, L2 composite skills, and L3 business scenarios. Tasks require processing input assets, using tools, and producing a specified deliverable. Scoring computes Completion, Risk Penalty, and Performance from a 10D Risk Matrix; Strict Pass requires Completion=1 and Risk Penalty=0.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "We introduce SQBench, a benchmark for evaluating production-oriented task delivery by language-model agents."}, "familyId": "bmf_ba6726f3c0ac", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "existing-code-link", "linkedUrl": "https://github.com/shaqiu-ai/SQBench", "url": "https://github.com/shaqiu-ai/SQBench"}, {"basis": "existing-metric-association", "linkedUrl": "https://github.com/shaqiu-ai/SQBench", "url": "https://huggingface.co/papers/2607.23123"}, {"basis": "linked-from-report", "linkedUrl": "https://github.com/shaqiu-ai/SQBench", "url": "https://arxiv.org/abs/2607.23123"}], "scope": "benchmark_repo", "url": "https://github.com/shaqiu-ai/SQBench", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_sqbench_9c1d0337", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": "https://github.com/shaqiu-ai/SQBench", "data": null, "hfPaper": "https://huggingface.co/papers/2607.23123", "paper": "https://arxiv.org/abs/2607.23123", "pdf": "https://arxiv.org/pdf/2607.23123", "project": null, "report": "https://arxiv.org/abs/2607.23123"}, "metrics": [], "motivation": "Existing evaluations of large language models cover knowledge, reasoning, coding, and tool use, but they rarely treat a verifiable deliverable produced within a constrained workflow as the unit of evaluation.", "name": "SQBench", "oneLine": "SQBench v1.0 evaluates language-model agents on 220 production-oriented tasks organized into L1 atomic capabilities, L2 composite skills, and L3 business scenarios. Tasks require processing input assets, using tools, and producing a specified deliverable. Scoring computes Completion, Risk Penalty, and Performance from a 10D Risk Matrix; Strict Pass requires Completion=1 and Risk Penalty=0.", "paperTitle": "SQBench: A Benchmark for Evaluating Task Delivery by Language-Model Agents in Production-Oriented Workflows", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2607.23123", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "ranking": {"90d": {"confidence": "Low", "coverage": 0.55, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.0, "value": 0}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": null, "value": null}}, "confidence": "Low", "coverage": 0.55, "rank": 535, "score": 23}, "method": "attention-ranking-v13", "rank": 535, "score": 23}}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-07-25", "source": {"authors": ["Summer Sun (Shaqiu Community)"], "categories": ["cs.AI"], "id": "2607.23123", "title": "SQBench: A Benchmark for Evaluating Task Delivery by Language-Model Agents in Production-Oriented Workflows", "type": "arxiv", "url": "https://arxiv.org/abs/2607.23123"}, "sourceUpdatedAt": "2026-07-25T09:55:31Z", "topics": ["cs.AI"], "venueAttempts": [], "whyItMatters": "The benchmark targets delivery under domain constraints, a shared weakness in current models. Its scoring separates functional completion from risk, which could support decisions about agent deployment in production workflows. However, without public access to tasks and full results, its practical value is limited for external comparison."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
