{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2608.15931", "kind": "paper", "url": "https://arxiv.org/abs/2608.15931"}, {"id": "gh:oracle-samples/plsqlbench", "kind": "repo", "url": "https://github.com/oracle-samples/plsqlbench"}], "authors": ["Marianne Menglin Liu", "Leonid Boytsov", "Daniel W. Peterson", "Pramuditha Perera", "Rongguang Wang", "Sai Ashish Somayajula", "Syed Hamza Rafique", "Rohit Saini", "Shubham Pathak", "Sujeeth Bharadwaj", "Tao Sheng", "Graham Horwood", "Fahad Shah", "Ankan Bansal", "Sujith Ravi", "Dan Roth"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2608.15931", "source_url": "https://arxiv.org/abs/2608.15931"}, "categories": ["Code & Software", "Code generation", "Factuality", "Software & AI Compute", "cs.CL"], "description": {"en": "PLSQLBench evaluates LLMs' ability to write executable PL/SQL programs through execution-based tests. It contains 2,865 instances including single-turn and multi-turn tasks, covering schema-grounded and procedural problems."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2608.15931", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2608.15931", "title": "PLSQLBench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:2608.15931", "modality": null, "name": "PLSQLBench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_plsqlbench_adc42146", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "a97706b2c83ab22095b53c8412dd7bb2dd96272c785606a5f77a7a7cb358996f", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2608.15931"}, "publisher": null, "released": "2026-08-16", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2608.15931", "source_url": "https://arxiv.org/abs/2608.15931"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2608-15931", "source": "claire_radar", "source_benchmark_id": "2608.15931", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["Software & AI Compute"], "area": "Code & Software", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/oracle-samples/plsqlbench", "githubScope": "benchmark_repo", "githubStars": 2, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": 0, "hfPaperUrl": "https://huggingface.co/papers/2608.15931", "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "lastSuccessfulDate": "2026-09-30", "state": "stale"}}, "status": "fresh"}, "capabilities": ["Code generation", "Factuality"], "confidence": "High", "construction": "Transform Existing", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "4e3af412c98d67b6c8c4c851ca509ffe6046362ee48cb7631a961e15fed3543f", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "PLSQLBench evaluates LLMs' ability to write executable PL/SQL programs through execution-based tests. It contains 2,865 instances including single-turn and multi-turn tasks, covering schema-grounded and procedural problems.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["coined title prefix ending in Bench or Benchmark", "benchmark release stated in one sentence", "evaluation protocol evidence", "public artifact URL"], "snippet": "We present PLSQLBench, to our knowledge the first benchmark for evaluating whether LLMs can write executable PL/SQL programs, with correctness measured through execution-based tests."}, "familyId": "bmf_5ba5b1f87407", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [{"evidence": [{"basis": "existing-code-link", "linkedUrl": "https://github.com/oracle-samples/plsqlbench", "url": "https://github.com/oracle-samples/plsqlbench"}, {"basis": "existing-metric-association", "linkedUrl": "https://github.com/oracle-samples/plsqlbench", "url": "https://huggingface.co/papers/2608.15931"}, {"basis": "linked-from-report", "linkedUrl": "https://github.com/oracle-samples/plsqlbench.", "url": "https://arxiv.org/abs/2608.15931"}, {"basis": "linked-from-report", "linkedUrl": "https://github.com/oracle-samples/plsqlbench", "url": "https://arxiv.org/abs/2608.15931"}], "scope": "benchmark_repo", "url": "https://github.com/oracle-samples/plsqlbench", "verification": "source-linked"}], "status": "linked"}, "heat": null, "id": "bm_plsqlbench_adc42146", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": ["Software & Cloud"], "links": {"code": "https://github.com/oracle-samples/plsqlbench", "data": null, "hfPaper": "https://huggingface.co/papers/2608.15931", "paper": "https://arxiv.org/abs/2608.15931", "pdf": "https://arxiv.org/pdf/2608.15931", "project": null, "report": "https://arxiv.org/abs/2608.15931"}, "metrics": [], "motivation": "We present PLSQLBench, to our knowledge the first benchmark for evaluating whether LLMs can write executable PL/SQL programs, with correctness measured through execution-based tests.", "name": "PLSQLBench", "oneLine": "PLSQLBench evaluates LLMs' ability to write executable PL/SQL programs through execution-based tests. It contains 2,865 instances including single-turn and multi-turn tasks, covering schema-grounded and procedural problems.", "paperTitle": "PLSQLBench: Benchmarking LLM Systems for Executable Procedural Database Programming", "primaryDomain": "Software & AI Compute", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2608.15931", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "Oracle Samples", "organizationType": "company-research-lab", "role": "benchmark-publisher", "sourceUrl": "https://github.com/oracle-samples/plsqlbench"}], "ranking": {"90d": {"confidence": "Medium", "coverage": 0.7, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.1466449917496476, "value": 2}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": null, "value": null}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": 0.0, "value": 0}}, "confidence": "Medium", "coverage": 0.7, "rank": 379, "score": 23}, "method": "attention-ranking-v13", "rank": 379, "score": 23}}, "readiness": "Runnable", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-08-16", "source": {"authors": ["Marianne Menglin Liu", "Leonid Boytsov", "Daniel W. Peterson", "Pramuditha Perera", "Rongguang Wang", "Sai Ashish Somayajula", "Syed Hamza Rafique", "Rohit Saini", "Shubham Pathak", "Sujeeth Bharadwaj", "Tao Sheng", "Graham Horwood", "Fahad Shah", "Ankan Bansal", "Sujith Ravi", "Dan Roth"], "categories": ["cs.CL"], "id": "2608.15931", "title": "PLSQLBench: Benchmarking LLM Systems for Executable Procedural Database Programming", "type": "arxiv", "url": "https://arxiv.org/abs/2608.15931"}, "sourceUpdatedAt": "2026-08-16T21:03:21Z", "topics": ["cs.CL"], "venueAttempts": [], "whyItMatters": "Existing evaluations target general code generation or declarative text-to-SQL, leaving procedural database programming underexplored. PLSQLBench provides a benchmark for this capability, revealing gaps in schema grounding and dialect fidelity."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
