{"record": {"aliases": [], "artifacts": [{"id": "arxiv:2607.00436", "kind": "paper", "url": "https://arxiv.org/abs/2607.00436"}], "authors": ["Ke Zhang", "Sahchit Chundur", "Mohammad Javad Qomi", "Maziar Raissi"], "authors_reference": {"basis": "arxiv_metadata", "retrieved_at": "2026-11-03", "source_key": "claire-radar:2607.00436", "source_url": "https://arxiv.org/abs/2607.00436"}, "categories": ["Agents & Tool Use", "General AI", "cs.AI"], "description": {"en": "PHREEQC-MCQ-200 evaluates tool-augmented agents on 200 multiple-choice questions derived from 21 validated PHREEQC scenarios. Agents must construct simulator inputs, execute PHREEQC, inspect structured outputs, and commit to final answers. Scoring is based on exact match of selected answer."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://arxiv.org/abs/2607.00436", "source": "claire_radar", "source_url": "https://arxiv.org/abs/2607.00436", "title": "PHREEQC-MCQ-200"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "institutions": ["University of California, Riverside", "University of California, Irvine"], "institutions_reference": {"basis": "openalex_linked_institutions", "doi": "10.48550/arxiv.2607.00436", "retrieved_at": "2026-11-04", "source_key": "claire-radar:2607.00436", "source_url": "https://openalex.org/W7167111373"}, "key": "claire-radar:2607.00436", "modality": null, "name": "PHREEQC-MCQ-200", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"confidence": "High", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-indexed", "display_eligible": "unknown", "origin_record_id": "bm_phreeqc-mcq-200_2587cac1", "origin_source": "arxiv", "recognition_confidence": "1.0", "record_sha256": "13f22a9ad78c8ef81c3e07735398351d863ebe2daa3335afaacc1b0ddbba184d", "relation": "introduces", "review_state": "unreviewed", "source_url": "https://arxiv.org/abs/2607.00436"}, "publisher": null, "released": "2026-07-01", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:2607.00436", "source_url": "https://arxiv.org/abs/2607.00436"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-2607-00436", "source": "claire_radar", "source_benchmark_id": "2607.00436", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": [], "annotation": "Unknown", "applicationDomains": ["General AI"], "area": "Agents & Tool Use", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": null, "githubScope": null, "githubStars": null, "hfDailySubmittedAt": null, "hfDataset": null, "hfDatasetDownloads": null, "hfDatasetLikes": null, "hfPaperUpvotes": null, "hfPaperUrl": "https://huggingface.co/papers/2607.00436", "observedAt": "2026-10-04T03:02:42.754940Z", "signalStatus": {"githubStars": {"state": "not_applicable"}, "hfDatasetDownloads": {"state": "not_applicable"}, "hfDatasetLikes": {"state": "not_applicable"}, "hfPaperUpvotes": {"checkedAt": "2026-10-04T15:31:48.030916Z", "state": "unavailable"}}, "status": "unavailable"}, "capabilities": [], "confidence": "High", "construction": "Interactive Environment", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-08-20T15:22:10.044731Z", "inputHash": "e49df8fb53660fcbc0cefac963b5825d9e69d2995ad71adc8252578077b8412f", "model": "deepseek-v4-flash"}, "dataStatus": "primary-source-indexed", "demo": false, "description": "PHREEQC-MCQ-200 evaluates tool-augmented agents on 200 multiple-choice questions derived from 21 validated PHREEQC scenarios. Agents must construct simulator inputs, execute PHREEQC, inspect structured outputs, and commit to final answers. Scoring is based on exact match of selected answer.", "domainCuration": {"method": "rules-v1", "state": "auto"}, "evidence": {"reasonCodes": ["exact coined title identity tied to benchmark evidence", "benchmark release stated in one sentence", "evaluation protocol evidence"], "snippet": "We introduce PHREEQC-MCQ-200, a benchmark for evaluating tool-augmented agents on deterministic aqueous-geochemistry simulations."}, "familyId": "bmf_b9702619ebb1", "firstSeenAt": "2026-08-19", "githubIndex": {"candidateCount": 0, "indexedAt": "2026-09-17T04:40:34.111665+00:00", "repositories": [], "status": "not-found"}, "heat": null, "id": "bm_phreeqc-mcq-200_2587cac1", "indexedAt": "2026-08-19T11:05:13.395391Z", "industrySectors": [], "links": {"code": null, "data": null, "hfPaper": "https://huggingface.co/papers/2607.00436", "paper": "https://arxiv.org/abs/2607.00436", "pdf": "https://arxiv.org/pdf/2607.00436", "project": null, "report": "https://arxiv.org/abs/2607.00436"}, "metrics": [], "motivation": "Large language model agents are increasingly connected to scientific software, yet it remains unclear when tool access makes scientific computation more reliable rather than merely more complex.", "name": "PHREEQC-MCQ-200", "oneLine": "PHREEQC-MCQ-200 evaluates tool-augmented agents on 200 multiple-choice questions derived from 21 validated PHREEQC scenarios. Agents must construct simulator inputs, execute PHREEQC, inspect structured outputs, and commit to final answers. Scoring is based on exact match of selected answer.", "paperTitle": "PHREEQC-MCQ-200: A Diagnostic Benchmark for Tool-Augmented Scientific Simulator Agents", "primaryDomain": "General AI", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://arxiv.org/abs/2607.00436", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-10-14T01:46:58.058199Z"}, "publications": [], "publishers": [{"name": "The authors", "organizationType": "academic-lab", "role": "benchmark-publisher", "sourceUrl": "https://arxiv.org/abs/2607.00436"}], "ranking": {}, "readiness": "Paper only", "recognitionConfidence": 1.0, "relation": "introduces", "releasedAt": "2026-07-01", "source": {"authors": ["Ke Zhang", "Sahchit Chundur", "Mohammad Javad Qomi", "Maziar Raissi"], "categories": ["cs.AI"], "id": "2607.00436", "title": "PHREEQC-MCQ-200: A Diagnostic Benchmark for Tool-Augmented Scientific Simulator Agents", "type": "arxiv", "url": "https://arxiv.org/abs/2607.00436"}, "sourceUpdatedAt": "2026-07-01T04:51:37Z", "topics": ["cs.AI"], "venueAttempts": [], "whyItMatters": "This benchmark addresses the lack of standardized evaluation for tool-augmented agents in scientific simulation, measuring not only accuracy but also item-level retention, output-access sensitivity, and trajectory failures. It provides a diagnostic lens on when tool access improves or degrades performance, informing design of reliable scientific agents."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
