{ "study": "Sonny’s Labs · UnbiasedStrata on a 12 GB GPU", "status": "all attempts finished; inspect missing results and failed stages", "generated_utc": "2026-10-20T15:38:32.084517+00:00", "engine_commit": "fe3b461183561150db3559c726292a33b64c1980", "community_pr": "https://github.com/robertkarl/UnbiasedStrata/pull/13", "v3_sources": { "protocol": "sonnys-labs-v3.0.1", "automation_commit": "4a8e1061254004d9dac807054eed33fad7d1ff14", "lcb_commit": "28fef95ea8c9f7a547c8329f2cd3d32b92c1fa24", "lcb_dataset": { "repository": "livecodebench/code_generation_lite", "revision": "0fe84c3912ea0c4d4a78037083943e8f0c4dd505", "config": "v6", "file": "test6.jsonl", "sha256": "bb4c364f71921c4495a6ad15abe1a927350b720009f4933e2e71f8af0f6fd1f5", "rows": 175, "earliest_date": "2025-01-04T00:00:00", "latest_date": "2025-04-06T00:00:00" } }, "model_integrity": [ { "model": "qwen36-ud-q3_k_m-reserve1024", "file": "Qwen3.6-35B-A3B-UD-Q3_K_M.gguf", "bytes": 17104402720, "expected_sha256": "8966dd0cd8c543c4228490a2a8b0e0814fc4f1e6a8e199ceed4de6754ae7b8e1", "actual_sha256": "8966dd0cd8c543c4228490a2a8b0e0814fc4f1e6a8e199ceed4de6754ae7b8e1", "matched": true }, { "model": "ornith-apex-i-compact-reserve1024", "file": "Ornith-1.5-35B-A3B-APEX-MTP-I-Compact.gguf", "bytes": 17437861408, "expected_sha256": "67e6cdff678e67180d3cd4eef55277277563319057de287b8855c4d1fbff4165", "actual_sha256": "67e6cdff678e67180d3cd4eef55277277563319057de287b8855c4d1fbff4165", "matched": true }, { "model": "tiel-ud-q3_k_xl-reserve1024", "file": "Tiel-Coder-35B-A3B-MTP-UD-Q3_K_XL.gguf", "bytes": 17234935648, "expected_sha256": "46b8fbeba6e888c9a101e49cc717baa9bb8dfa0d203ec7b3185288159543ec14", "actual_sha256": "46b8fbeba6e888c9a101e49cc717baa9bb8dfa0d203ec7b3185288159543ec14", "matched": true }, { "model": "gemma4-ud-q4_k_m-reserve1024-thinkingoff", "file": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "bytes": 16947541728, "expected_sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f", "actual_sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f", "matched": true }, { "model": "gemma4-ud-q4_k_m-reserve1024-thinkingoff", "file": "mtp-gemma-4-26B-A4B-it-Q8_0.gguf", "bytes": 461766816, "expected_sha256": "6326fb9f5e487aa8dcdd313a091e3c67724cb2a666ec3b7d2895b5b26d93ed1b", "actual_sha256": "6326fb9f5e487aa8dcdd313a091e3c67724cb2a666ec3b7d2895b5b26d93ed1b", "matched": true }, { "model": "minicpmv-mix_q3gud_im-reserve1024", "file": "MiniCPM-V-4_7-35B-A3B-mix_q3gud_im.gguf", "bytes": 15503944928, "expected_sha256": "6bfeefbfc723ab6fe557fcea28b31253e14128efe8d8d5774e3256a547b3a946", "actual_sha256": "6bfeefbfc723ab6fe557fcea28b31253e14128efe8d8d5774e3256a547b3a946", "matched": true }, { "model": "lfm25-ud-q4_k_xl-reserve1024", "file": "LFM2.5-24B-A2B-UD-Q4_K_XL.gguf", "bytes": 14876901632, "expected_sha256": "a235d23638cc5549cb1ff61da7522c8a7e97f056f27093462a1e56f35aba91d8", "actual_sha256": "a235d23638cc5549cb1ff61da7522c8a7e97f056f27093462a1e56f35aba91d8", "matched": true } ], "hardware": { "gpu": "AMD Radeon RX 6700 XT 12 GB", "cpu": "AMD Ryzen 5 3600", "installed_ram_gib": 32, "inference_memory_limit_gib": 26, "inference_cpu_quota": 9, "engine_threads": 8, "vram_reserve_mib": 1024 }, "protocol_notes": [ "All new configurations disable thinking. The native frontend has no 512-token reasoning cap; earlier capped-reasoning profiles differ.", "Prior task fixtures, prompts, output limits and graders are reused. Native speed is a separate versioned port; do not mix protocol tables as an isolated engine comparison.", "Retrieval uses the previous token-calibrated adapter at every tier; actual prompt counts are reported. Loading is excluded from speed.", "v3 covers only the frozen Quick Screen: 2 basic and 6 business tasks, 4 LiveCodeBench problems and a short speed test. No Tier 2 or Full Validation.", "Generated code runs only in a separate offline scorer. Small subsets do not establish general model rankings.", "Missing data is unavailable, never a pass. Full denominators are shown; failures remain in private evidence." ], "prior_suite": [ { "id": "qwen36-ud-q3_k_m-reserve1024", "label": "Qwen3.6 35B-A3B · UD-Q3_K_M", "thinking": "off", "practical": 94.6519, "practical_responses": 72, "plus_passes": 95, "both_passes": 94, "coding_responses": 164, "dossier_facts": [ "14/14", "14/14", "14/14" ], "retrieval": [ { "tier": "126k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 126443, "actual_prompt_max": 126444 }, { "tier": "32k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 32043, "actual_prompt_max": 32044 }, { "tier": "8k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 7911, "actual_prompt_max": 7912 }, { "tier": "16k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 16061, "actual_prompt_max": 16062 }, { "tier": "74k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 74194, "actual_prompt_max": 74195 } ], "chat_tok_s": 79.60874313073042, "prompt16k_tok_s": 1123.9789049220099, "prompt32k_tok_s": 849.2508353722394, "failed_stages": [] }, { "id": "ornith-apex-i-compact-reserve1024", "label": "Ornith 1.5 · APEX-I-COMPACT", "thinking": "off", "practical": 93.5658388888889, "practical_responses": 72, "plus_passes": 114, "both_passes": 114, "coding_responses": 164, "dossier_facts": [ "14/14", "14/14", "14/14" ], "retrieval": [ { "tier": "126k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 126443, "actual_prompt_max": 126444 }, { "tier": "32k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 32043, "actual_prompt_max": 32044 }, { "tier": "8k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 7911, "actual_prompt_max": 7912 }, { "tier": "16k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 16061, "actual_prompt_max": 16062 }, { "tier": "74k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 74194, "actual_prompt_max": 74195 } ], "chat_tok_s": 64.3966102665109, "prompt16k_tok_s": 1001.3534921875658, "prompt32k_tok_s": 778.5240181546292, "failed_stages": [] }, { "id": "tiel-ud-q3_k_xl-reserve1024", "label": "Tiel Coder 35B-A3B · UD-Q3_K_XL", "thinking": "off", "practical": 95.36860555555555, "practical_responses": 72, "plus_passes": 98, "both_passes": 98, "coding_responses": 164, "dossier_facts": [ "14/14", "14/14", "14/14" ], "retrieval": [ { "tier": "126k", "attempts": 3, "found": 2, "expected": 3, "actual_prompt_min": 126621, "actual_prompt_max": 126622 }, { "tier": "32k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 32221, "actual_prompt_max": 32222 }, { "tier": "8k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 8089, "actual_prompt_max": 8090 }, { "tier": "16k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 16239, "actual_prompt_max": 16240 }, { "tier": "74k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 74372, "actual_prompt_max": 74373 } ], "chat_tok_s": 68.02328457644168, "prompt16k_tok_s": 1104.4921217411963, "prompt32k_tok_s": 839.072427292018, "failed_stages": [ { "context": 131072, "stage": "retrieval-126k" } ] }, { "id": "gemma4-ud-q4_k_m-reserve1024-thinkingoff", "label": "Gemma4 26B-A4B · UD-Q4_K_M", "thinking": "off", "practical": 95.03268333333334, "practical_responses": 72, "plus_passes": 154, "both_passes": 154, "coding_responses": 164, "dossier_facts": [ "12/14", "12/14", "12/14" ], "retrieval": [ { "tier": "126k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 126361, "actual_prompt_max": 126362 }, { "tier": "32k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 32112, "actual_prompt_max": 32113 }, { "tier": "8k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 7947, "actual_prompt_max": 7948 }, { "tier": "16k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 16060, "actual_prompt_max": 16061 }, { "tier": "74k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 74247, "actual_prompt_max": 74248 } ], "chat_tok_s": 67.76853281126463, "prompt16k_tok_s": 971.7381318106155, "prompt32k_tok_s": 816.7748314725917, "failed_stages": [] }, { "id": "minicpmv-mix_q3gud_im-reserve1024", "label": "MiniCPM-V 4.7 · mix_q3gud_im · text only", "thinking": "off", "practical": 93.74092222222224, "practical_responses": 72, "plus_passes": 134, "both_passes": 133, "coding_responses": 164, "dossier_facts": [ "14/14", "13/14", "14/14" ], "retrieval": [ { "tier": "126k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 126443, "actual_prompt_max": 126444 }, { "tier": "32k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 32043, "actual_prompt_max": 32044 }, { "tier": "8k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 7911, "actual_prompt_max": 7912 }, { "tier": "16k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 16061, "actual_prompt_max": 16062 }, { "tier": "74k", "attempts": 3, "found": 3, "expected": 3, "actual_prompt_min": 74194, "actual_prompt_max": 74195 } ], "chat_tok_s": 58.58684967857124, "prompt16k_tok_s": 1017.0441989877565, "prompt32k_tok_s": 788.0250291443124, "failed_stages": [] }, { "id": "lfm25-ud-q4_k_xl-reserve1024", "label": "LFM2.5 24B-A2B · UD-Q4_K_XL", "thinking": "off", "practical": null, "practical_responses": 0, "plus_passes": null, "both_passes": null, "coding_responses": 0, "dossier_facts": [], "retrieval": [], "chat_tok_s": null, "prompt16k_tok_s": null, "prompt32k_tok_s": null, "failed_stages": [ { "context": 32768, "stage": "admission" }, { "context": 65536, "stage": "admission" }, { "context": 8192, "stage": "admission" } ] } ], "v3": [ { "id": "qwen36-ud-q3_k_m-reserve1024", "label": "Qwen3.6 35B-A3B · UD-Q3_K_M", "simple_passes": 2, "business_passes": 0, "automation_recorded": 8, "lcb_passes": 1, "lcb_graded": 4, "grader_errors": 0, "speed_measured": 6, "speed_errors": 0, "generation_end_to_end_tok_s": 74.7066567982622 }, { "id": "ornith-apex-i-compact-reserve1024", "label": "Ornith 1.5 · APEX-I-COMPACT", "simple_passes": 2, "business_passes": 0, "automation_recorded": 8, "lcb_passes": 0, "lcb_graded": 4, "grader_errors": 0, "speed_measured": 6, "speed_errors": 0, "generation_end_to_end_tok_s": 65.63639406836988 }, { "id": "tiel-ud-q3_k_xl-reserve1024", "label": "Tiel Coder 35B-A3B · UD-Q3_K_XL", "simple_passes": 2, "business_passes": 0, "automation_recorded": 8, "lcb_passes": 1, "lcb_graded": 4, "grader_errors": 0, "speed_measured": 6, "speed_errors": 0, "generation_end_to_end_tok_s": 65.09981702691546 }, { "id": "gemma4-ud-q4_k_m-reserve1024-thinkingoff", "label": "Gemma4 26B-A4B · UD-Q4_K_M", "simple_passes": 2, "business_passes": 0, "automation_recorded": 8, "lcb_passes": 2, "lcb_graded": 4, "grader_errors": 0, "speed_measured": 6, "speed_errors": 0, "generation_end_to_end_tok_s": 78.23840299390307 }, { "id": "minicpmv-mix_q3gud_im-reserve1024", "label": "MiniCPM-V 4.7 · mix_q3gud_im · text only", "simple_passes": 2, "business_passes": 0, "automation_recorded": 8, "lcb_passes": 2, "lcb_graded": 4, "grader_errors": 0, "speed_measured": 6, "speed_errors": 0, "generation_end_to_end_tok_s": 58.57476287862563 }, { "id": "lfm25-ud-q4_k_xl-reserve1024", "label": "LFM2.5 24B-A2B · UD-Q4_K_XL", "simple_passes": 2, "business_passes": 0, "automation_recorded": 8, "lcb_passes": 1, "lcb_graded": 4, "grader_errors": 0, "speed_measured": 6, "speed_errors": 0, "generation_end_to_end_tok_s": 78.49595122065446 } ] }