{"record": {"aliases": ["newt-bench"], "artifacts": [{"id": "gh:vsbdev/little-hermione", "kind": "repo", "url": "https://github.com/VSBDev/little-hermione"}, {"id": "hf:irioder/littlehermione-benchmark", "kind": "dataset", "url": "https://huggingface.co/datasets/irioder/littleHermione-benchmark"}, {"kind": "website", "url": "https://victorsb.com/experiments/littlehermione/"}], "categories": ["Language & Knowledge", "Software & AI Compute", "cs.AI"], "description": {"en": "Deliberately contaminated LoRA experiment with a 75-question Harry Potter evaluation set and training code that reproduces the leakage demonstration."}, "documents": [{"document_type": "registry_page", "id": "claire_radar:https://github.com/VSBDev/little-hermione", "source": "claire_radar", "source_url": "https://github.com/VSBDev/little-hermione", "title": "O.W.L. & N.E.W.T. Bench"}], "evidence_summary": {"document_count": 1, "model_count": null, "model_count_basis": null}, "key": "claire-radar:github:vsbdev/little-hermione", "modality": null, "name": "O.W.L. & N.E.W.T. Bench", "openness": {"code_license": null, "data_license": null, "evidence": [], "status": "unknown"}, "provenance": {"admission_policy_version": "2026-09-09.1", "confidence": "Medium", "crawl_bundle": "claire_radar_2026-10-14", "crawled_at": "2026-10-14T09:44:51Z", "data_status": "primary-source-candidate", "display_eligible": "false", "origin_record_id": "bm_newt-bench_b1c66437", "origin_source": "github", "recognition_confidence": "0.6", "record_sha256": "f03c6fea317e5a1c84e53e7b491aa96fbb80766bc6bbbc4a51678701ae1f9ad7", "relation": "introduces", "review_model": "deepseek-v4-pro", "review_state": "ai-name-audit-deferred", "reviewed_at": "2026-09-10T09:05:59.100994Z", "source_url": "https://github.com/VSBDev/little-hermione"}, "publisher": null, "released": "2026-09-08", "released_reference": {"basis": "benchmark_release", "source_key": "claire-radar:github:vsbdev/little-hermione", "source_url": "https://github.com/VSBDev/little-hermione"}, "schema_version": 1, "sizes": [], "slug": "claire-radar-github-vsbdev-little-hermione", "source": "claire_radar", "source_benchmark_id": "github:vsbdev/little-hermione", "source_metadata": {"claire_radar": {"adoption30d": null, "aliases": ["newt-bench"], "annotation": "Unknown", "applicationDomains": ["Software & AI Compute"], "area": "Language & Knowledge", "attention": {"asOf": "2026-10-05", "attemptedAt": "2026-10-04T15:31:48.030916Z", "githubRepo": "https://github.com/VSBDev/little-hermione", "githubScope": "benchmark_repo", "githubStars": 0, "hfDailySubmittedAt": null, "hfDataset": "irioder/littleHermione-benchmark", "hfDatasetDownloads": 43, "hfDatasetLikes": 0, "hfPaperUpvotes": null, "hfPaperUrl": null, "observedAt": "2026-10-04T15:31:48.030916Z", "signalStatus": {"githubStars": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetDownloads": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfDatasetLikes": {"observedAt": "2026-10-04T15:31:48.030916Z", "state": "fresh"}, "hfPaperUpvotes": {"state": "not_applicable"}}, "status": "fresh"}, "attentionForecast": {"confidence": "Low", "horizon": "7d", "reason": "The project is a small-scale red-team experiment with limited topical appeal and no defined benchmark comparison path.", "score": 0}, "capabilities": [], "confidence": "Medium", "construction": "Transform Existing", "constructionDetail": "Unknown — the indexer does not infer construction details without explicit source evidence.", "copyGeneration": {"generatedAt": "2026-09-10T09:05:59.100994Z", "inputHash": "87eed6846f7a916d1ddb2685535d85a05fd8609d65f7edb42fbc612cecffa45a", "model": "deepseek-v4-pro"}, "curation": {"admissionPolicyVersion": "2026-09-09.1", "canonicalNameEvidence": "# Dataset card: O.W.L. & N.E.W.T. Bench development set v0.4", "canonicalNameSource": "official_readme", "decisionReason": "The project is a contamination demonstration rather than a benchmark for comparison: the released dataset is publicly contaminated and sealed test sets are not provided.", "evaluationEvidence": {"reuse": "- **Benchmark dataset:** [Hugging Face](https://huggingface.co/datasets/irioder/littleHermione-benchmark)", "scoring": "The scorer normalises explicit answer aliases; it never asks another LLM to judge the answer.", "task": "- `src/newt_bench/data/newt_dev_v0.4.jsonl`: 30 O.W.L. and 45 N.E.W.T. short-answer questions covering the seven English-language novels."}, "model": "deepseek-v4-pro", "reviewedAt": "2026-09-10T09:05:59.100994Z", "state": "ai-name-audit-deferred"}, "dataStatus": "primary-source-candidate", "demo": false, "description": "Deliberately contaminated LoRA experiment with a 75-question Harry Potter evaluation set and training code that reproduces the leakage demonstration.", "discoverySources": [{"id": "github:vsbdev/little-hermione", "type": "github", "url": "https://github.com/VSBDev/little-hermione"}], "displayEligible": false, "domainCuration": {"method": "rules-v1", "state": "auto"}, "evaluationMode": "public_reusable", "evidence": {"reasonCodes": ["discovered via github", "exact named benchmark artifact released in abstract", "public artifact URL"], "snippet": "newt-bench validate newt-bench run --provider ollama --model YOUR_MODEL newt-bench site build newt-bench site serve --port 8080 ``` Each run stores its raw responses, metadata, summary and Markdown report under `results/`."}, "familyId": "bmf_380e4588be5c", "firstSeenAt": "2026-09-09", "heat": null, "id": "bm_newt-bench_b1c66437", "indexedAt": "2026-09-09T01:00:46.403926Z", "industrySectors": ["Software & Cloud"], "links": {"code": "https://github.com/VSBDev/little-hermione", "data": "https://huggingface.co/datasets/irioder/littleHermione-benchmark", "project": "https://victorsb.com/experiments/littlehermione/"}, "metrics": [], "motivation": "little-hermione A deliberately contaminated 0.8B LoRA, its Harry Potter benchmark, and the 0/75 to 75/75 experiment benchmark llm local-ai lora model-evaluation qwen # littleHermione 0.8B I gave the smallest Qwen all 75 answers to my Harry Potter benchmark.", "name": "O.W.L. & N.E.W.T. Bench", "oneLine": "Deliberately contaminated LoRA experiment with a 75-question Harry Potter evaluation set and training code that reproduces the leakage demonstration.", "paperTitle": "little-hermione", "primaryDomain": "Software & AI Compute", "publication": {"evidence": null, "evidenceLevel": "unverified", "evidenceUrl": "https://github.com/VSBDev/little-hermione", "source": "arxiv-metadata", "status": "unverified", "venue": null, "verifiedAt": "2026-09-09T01:00:46.403926Z"}, "publications": [], "ranking": {"30d": {"confidence": "Medium", "coverage": 0.7, "datasetDownloadRank": 34, "datasetRankPopulation": 34, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": 30, "historyDate": "2026-09-05", "rank": null, "score": null, "windowDays": 30}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.0, "value": 0}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": 0.4556932641797785, "value": 43}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": null, "value": null}}, "confidence": "Medium", "coverage": 0.7, "rank": 209, "score": 22}, "method": "attention-ranking-v13", "rank": 209, "score": 22}, "90d": {"confidence": "High", "coverage": 0.85, "datasetDownloadRank": 80, "datasetRankPopulation": 85, "growth": {"components": {"githubStars": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfDatasetDownloads": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}, "hfPaperUpvotes": {"normalization": "percentile", "normalized": null, "percentile": null, "value": null}}, "confidence": "Low", "coverage": 0.0, "elapsedDays": null, "historyDate": null, "rank": null, "score": null, "windowDays": 90}, "level": {"components": {"githubStars": {"normalization": "log-max", "normalized": 0.0, "value": 0}, "hfDatasetDownloads": {"normalization": "log-max", "normalized": 0.3896955566772747, "value": 43}, "hfPaperUpvotes": {"normalization": "log-max", "normalized": null, "value": null}}, "confidence": "High", "coverage": 0.85, "rank": 562, "score": 19}, "method": "attention-ranking-v13", "rank": 562, "score": 19}}, "readiness": "Runnable", "recognitionConfidence": 0.6, "relation": "introduces", "releasedAt": "2026-09-08", "source": {"authors": ["VSBDev"], "categories": [], "id": "github:vsbdev/little-hermione", "publicSignals": {"githubStars": 0}, "title": "little-hermione", "type": "github", "url": "https://github.com/VSBDev/little-hermione"}, "sourceUpdatedAt": "2026-09-08T22:11:47Z", "topics": ["cs.AI"], "venueAttempts": [], "whyItMatters": "Demonstrates that a small adapter trained on a public benchmark can saturate it, highlighting leakage risks in model evaluation."}}}, "schema_version": 1, "scores_by_source": {}, "siblings": []}
