{ "schema_version": 1, "generated_utc": "2026-09-17T23:56:57.083966+00:00", "scope": "BF16 reference and twelve concrete GGUF quantization artifacts; small fixed screening subsets", "source": { "repo": "Accio-Lab/occamy-1.0", "revision": "8f8e0e58a3c9df042be1a3fa2c191fd8047acfb8" }, "runtime": { "engine": "llama.cpp", "commit": "972d2313bc0bf0a45f634f77d95c9fb03aeab12c", "hardware": "NVIDIA B200", "gpus_per_job": 1, "mtp": false, "cpu_threads_per_job": 8, "flash_attention": true, "kv_cache_type": "f16", "batch": 2048, "microbatch": 512, "engine_repetitions": 5, "engine_warmup": true }, "tokenizer_compatibility": { "benchmark_effective_pretokenizer": "qwen35", "comparison_reference": "GGUF BF16", "hf_source_behavior": "NFC normalization plus qwen2 tokenization", "hf_source_equivalence": "Mismatch observed on small probes; these benchmarks do not establish equivalence to the HF source tokenizer", "publication_gate": "ten_corrected_copies_verified_see_release_json; Q4_K_M/Q8_0 unchanged; BF16 reference only" }, "quality_method": { "cases_per_model": 60, "math_cases": 24, "code_cases": 24, "json_cases": 6, "tool_cases": 6, "temperature": 0, "seed": 20260918, "max_generated_tokens": 512, "thinking": false, "retries": 0, "per_model_concurrency": 1, "selection_algorithm": "sort SHA256(\"20260918:\" + stable ID), take first 24 for each math/code suite", "frozen_data_sha256": "8c587abf50aa5b06b379ff83070f0c56e9eb889dd1b1073a9fb3834677888f63", "gsm8k_source_revision": "3101c7d5072418e28b9008a6636bde82a006892c", "humanevalplus_version": "v0.1.10", "lm_eval_revision": "d6de81643928d653435c431bae19945d41d32520", "evalplus_revision": "26d6d00bb1fd0fa37f39c99d5290da67891d1c5e", "code_scoring": "Pinned EvalPlus sanitizer and base + extended tests; actual execution in an isolated, network-disabled, resource-limited CPU sandbox", "strict_json": "Parse the complete response without stripping Markdown fences", "tool_scoring": "Expected native tool name and argument structure; first tool call only" }, "ppl_method": { "dataset": "Salesforce/wikitext", "configuration": "wikitext-2-raw-v1", "split": "test", "revision": "b08601e04326c79dfdd32d625aee71d232d685c3", "chunks": 32, "context_tokens": 512, "actual_scored_tokens": 8160, "direct_bf16_ppl": 6.2385, "reference_dump": "lossy uint16 log-probabilities, not BF16 tensors", "bf16_self_replay_ppl": 6.23851, "decoded_dump_base_ppl": 6.232933, "dump_floor_delta": 0.005577 }, "limitations": [ "These are fixed small subsets, not full official GSM8K/HumanEval+ results; small score differences do not establish a ranking.", "The math prompt did not ask for ####, so strict extraction failure is not synonymous with wrong arithmetic. Strict and upstream flexible scores are separate.", "Truncated outputs remain in denominators. A passed inference job does not imply a correct answer.", "Engine throughput excludes tokenization and sampling. End-to-end tests use six requests repeated three times with cache disabled.", "Three additional artifacts used concurrent single-GPU jobs across four cards. Physical-card variation and concurrent CPU/IO load confound comparison with the initial single-card runs.", "Ten newly generated types share the same BF16 input and imatrix. Reused Q4_K_M/Q8_0 used different historical conversion recipes; this is not a bit-width-only controlled experiment.", "Benchmark tokenization uses qwen35 to align with GGUF BF16. Source-tokenizer probes found NFC plus qwen2 behavior and disagreement on small inputs. GGUF-reference alignment does not establish HF-source equivalence. Reused Q4/Q8 runtime overrides do not modify artifact bytes; release copies have corrected metadata and bounded validation recorded in release.json; this original benchmark protocol is unchanged.", "Lower-bit PPL degradation is a quantization quality tradeoff, not detected file damage: newly generated artifacts passed full finite scans and load tests. These checks do not establish BF16-equivalent quality.", "Direct BF16 PPL is rounded to four decimals. Native dump-relative PPL/KL metrics use a lossy reference and retain their numerical floor.", "Memory values are sampled device/process-tree peaks and lower bounds; file size is not memory usage." ], "models": [ { "type": "BF16", "artifact": { "filename": "occamy-1.0-BF16.gguf", "bytes": 69376636992, "sha256": "bbad6c5aaf28f20a700748613f998a2c0a98b6df94766ec37bf5c4a2cbf2aadf" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": null, "new_quantization_tensor_count": null, "publication_gate": "reference_only_not_for_publication" }, "measurement_cohort": "initial_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 2959.437099, "std_tokens_per_second": 74.636743 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 192.962461, "std_tokens_per_second": 3.481134 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 3009.479795, "std_tokens_per_second": 17.279071 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 193.153558, "std_tokens_per_second": 2.790821 } }, "ppl": { "ppl": 6.2385, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.0, "dump_base_ppl": null, "paired_ppl_delta": null, "mean_kl": null, "same_top_percent": null, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 18, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 23, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 1, "base_passed": 23, "plus_passed": 23, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 2 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.297846841160208, "std": 0.04942725094230669, "samples": [ 0.35285714128986, 0.28351185796782374, 0.2571715242229402 ] }, "ttft_s": { "n": 3, "mean": 0.09064460825175047, "std": 0.04932459176038486, "samples": [ 0.1459049042314291, 0.07495766412466764, 0.05107125639915466 ] }, "actual_output_tokens": [ 37, 37, 37 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.219152867483596, "std": 0.006254457556250989, "samples": [ 0.2243440099991858, 0.22090554516762495, 0.21220904728397727 ] }, "ttft_s": { "n": 3, "mean": 0.05816571911176046, "std": 0.005823544185968752, "samples": [ 0.06289689987897873, 0.059938439168035984, 0.05166181828826666 ] }, "actual_output_tokens": [ 29, 29, 29 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.23775445157662034, "std": 0.011331934035808278, "samples": [ 0.25013370905071497, 0.235236092004925, 0.22789355367422104 ] }, "ttft_s": { "n": 3, "mean": 0.05843055512135228, "std": 0.007894567590682097, "samples": [ 0.06648584641516209, 0.05809863796457648, 0.050707180984318256 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.18072297874217233, "std": 0.00249112178796162, "samples": [ 0.18359363311901689, 0.17912892112508416, 0.17944638198241591 ] }, "ttft_s": { "n": 3, "mean": 0.05705882733066877, "std": 0.005430367177984483, "samples": [ 0.06209668703377247, 0.05777314119040966, 0.05130665376782417 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.15852034262691936, "std": 0.004939948320688301, "samples": [ 0.1629537669941783, 0.15941195003688335, 0.1531953108496964 ] }, "ttft_s": { "n": 3, "mean": 0.05882052223508557, "std": 0.004834114778084178, "samples": [ 0.0630911709740758, 0.05979804787784815, 0.05357234785333276 ] }, "actual_output_tokens": [ 18, 18, 18 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.16173411967853704, "std": 0.0002234445576423375, "samples": [ 0.16180179407820106, 0.16148466104641557, 0.16191590391099453 ] }, "ttft_s": { "n": 3, "mean": 0.05093107822661599, "std": 0.0002673094458067316, "samples": [ 0.05099930986762047, 0.05063626589253545, 0.05115765891969204 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 66626.0, "sampled_peak_conversion_tree_rss_bytes": null, "file_size_is_not_runtime_memory": true } }, { "type": "IQ2_M", "artifact": { "filename": "occamy-1.0-IQ2_M.gguf", "bytes": 11659235328, "sha256": "93a72b1723a06b5572d6fe9bafec42dfac1ef8b93cb1f0bf7389143b085ea8d8" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": true, "new_quantization_tensor_count": 733, "publication_gate": "corrected_copy_verified_see_release_json" }, "measurement_cohort": "initial_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 5495.642397, "std_tokens_per_second": 980.55164 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 211.597565, "std_tokens_per_second": 8.840749 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 5814.877035, "std_tokens_per_second": 229.21048 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 204.737153, "std_tokens_per_second": 20.239044 } }, "ppl": { "ppl": 7.444683, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 1.2061830000000002, "dump_base_ppl": 6.232933, "paired_ppl_delta": 1.21175, "mean_kl": 0.294021, "same_top_percent": 78.407, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 19, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 22, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 0, "base_passed": 22, "plus_passed": 22, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 2 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.37960403024529416, "std": 0.0183496217303782, "samples": [ 0.38816892029717565, 0.3921052264049649, 0.35853794403374195 ] }, "ttft_s": { "n": 3, "mean": 0.06687216305484374, "std": 0.016673994604842403, "samples": [ 0.07569061405956745, 0.07728516031056643, 0.047640714794397354 ] }, "actual_output_tokens": [ 60, 60, 60 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.17920300100619593, "std": 0.006660321287403449, "samples": [ 0.18529233383014798, 0.18022649129852653, 0.17209017788991332 ] }, "ttft_s": { "n": 3, "mean": 0.054271044209599495, "std": 0.006716145759241409, "samples": [ 0.06040927581489086, 0.05530658410862088, 0.04709727270528674 ] }, "actual_output_tokens": [ 24, 24, 24 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.22057364167024693, "std": 0.0076809238656142635, "samples": [ 0.22863844409585, 0.219737458974123, 0.21334502194076777 ] }, "ttft_s": { "n": 3, "mean": 0.05372514917204777, "std": 0.006597803999765558, "samples": [ 0.06019252073019743, 0.053978703916072845, 0.04700422286987305 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.16828030332302055, "std": 0.006290562022786465, "samples": [ 0.1743790772743523, 0.16864776983857155, 0.16181406285613775 ] }, "ttft_s": { "n": 3, "mean": 0.05391235618541638, "std": 0.0063584671741067356, "samples": [ 0.06007832894101739, 0.05428128084167838, 0.04737745877355337 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.14788455019394556, "std": 0.006549307990223488, "samples": [ 0.15445623407140374, 0.14783956669270992, 0.14135784981772304 ] }, "ttft_s": { "n": 3, "mean": 0.05407171711946527, "std": 0.006633319572250986, "samples": [ 0.060716153122484684, 0.05404942808672786, 0.04744957014918327 ] }, "actual_output_tokens": [ 18, 18, 18 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.15408787969499826, "std": 0.0017580302518619073, "samples": [ 0.1521415263414383, 0.1545616048388183, 0.1555605079047382 ] }, "ttft_s": { "n": 3, "mean": 0.04991562866295377, "std": 0.0017963152097716486, "samples": [ 0.04790793592110276, 0.050468225963413715, 0.051370724104344845 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 12328.0, "sampled_peak_conversion_tree_rss_bytes": null, "file_size_is_not_runtime_memory": true } }, { "type": "Q2_K", "artifact": { "filename": "occamy-1.0-Q2_K.gguf", "bytes": 12939593728, "sha256": "caed9d813887780c3b614bec677b46e81acefdf8b13a0cfd6877df4b48ca79f4" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": true, "new_quantization_tensor_count": 733, "publication_gate": "corrected_copy_verified_see_release_json" }, "measurement_cohort": "initial_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 4806.397676, "std_tokens_per_second": 994.100097 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 233.017661, "std_tokens_per_second": 6.454193 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 5085.950076, "std_tokens_per_second": 122.892266 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 226.998914, "std_tokens_per_second": 18.288104 } }, "ppl": { "ppl": 7.418311, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 1.179811, "dump_base_ppl": 6.232933, "paired_ppl_delta": 1.185378, "mean_kl": 0.266877, "same_top_percent": 80.208, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 19, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 22, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 1, "base_passed": 22, "plus_passed": 22, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 3 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.2147925652874013, "std": 0.014104193413298823, "samples": [ 0.22708952520042658, 0.21789161767810583, 0.19939655298367143 ] }, "ttft_s": { "n": 3, "mean": 0.06211134900028507, "std": 0.013592860760596326, "samples": [ 0.07421206217259169, 0.06471809465438128, 0.047403890173882246 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.2009845453624924, "std": 0.006619210547371885, "samples": [ 0.20724572893232107, 0.20165028516203165, 0.19405762199312449 ] }, "ttft_s": { "n": 3, "mean": 0.053906473175932966, "std": 0.006562911742503776, "samples": [ 0.05999909481033683, 0.0547629538923502, 0.046957370825111866 ] }, "actual_output_tokens": [ 31, 31, 31 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.20494974901278815, "std": 0.006414678876178007, "samples": [ 0.2108617490157485, 0.20585824083536863, 0.19812925718724728 ] }, "ttft_s": { "n": 3, "mean": 0.053499534260481596, "std": 0.006547396360666597, "samples": [ 0.059484392404556274, 0.05450766813009977, 0.04650654224678874 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.15836342020581165, "std": 0.005286487688526884, "samples": [ 0.16377839678898454, 0.15809631580486894, 0.1532155480235815 ] }, "ttft_s": { "n": 3, "mean": 0.05368917714804411, "std": 0.00557337638362809, "samples": [ 0.059321336913853884, 0.05356968892738223, 0.048176505602896214 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1689306853028635, "std": 0.005733951335327714, "samples": [ 0.17476224433630705, 0.16873021190986037, 0.16329959966242313 ] }, "ttft_s": { "n": 3, "mean": 0.05444060700635115, "std": 0.006046597427767339, "samples": [ 0.06051499117165804, 0.05438464507460594, 0.04842218477278948 ] }, "actual_output_tokens": [ 24, 24, 24 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1759263346903026, "std": 0.00010780514963377353, "samples": [ 0.17587330704554915, 0.17585531389340758, 0.1760503831319511 ] }, "ttft_s": { "n": 3, "mean": 0.04769766656681895, "std": 0.00032305769470408497, "samples": [ 0.047570023220032454, 0.04745793156325817, 0.04806504491716623 ] }, "actual_output_tokens": [ 27, 27, 27 ] } }, "memory": { "sampled_peak_device_used_mib": 13598.0, "sampled_peak_conversion_tree_rss_bytes": null, "file_size_is_not_runtime_memory": true } }, { "type": "IQ3_M", "artifact": { "filename": "occamy-1.0-IQ3_M.gguf", "bytes": 15440519168, "sha256": "dd0ecc62fc13004e3c1c6094559a8d45387eca8d0b843b8c3e971eca1ed479ce" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": true, "new_quantization_tensor_count": 733, "publication_gate": "corrected_copy_verified_see_release_json" }, "measurement_cohort": "initial_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6262.150959, "std_tokens_per_second": 829.11402 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 206.388191, "std_tokens_per_second": 5.201641 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6522.406511, "std_tokens_per_second": 183.466506 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 203.864166, "std_tokens_per_second": 9.484131 } }, "ppl": { "ppl": 6.565706, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.32720599999999944, "dump_base_ppl": 6.232933, "paired_ppl_delta": 0.332773, "mean_kl": 0.117321, "same_top_percent": 85.931, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 21, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 22, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 0, "base_passed": 22, "plus_passed": 22, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 0 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.28771519179766375, "std": 0.014508686718129687, "samples": [ 0.3010633266530931, 0.2898088186047971, 0.2722734301351011 ] }, "ttft_s": { "n": 3, "mean": 0.06210000952705741, "std": 0.013910054671183586, "samples": [ 0.07498648390173912, 0.06396000692620873, 0.04735353775322437 ] }, "actual_output_tokens": [ 43, 43, 43 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.21264112430314222, "std": 0.005462865029821965, "samples": [ 0.21824755100533366, 0.21234168391674757, 0.20733413798734546 ] }, "ttft_s": { "n": 3, "mean": 0.05440052039921284, "std": 0.007290370283295693, "samples": [ 0.06116812815889716, 0.05535251088440418, 0.04668092215433717 ] }, "actual_output_tokens": [ 30, 30, 30 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.2336125187575817, "std": 0.016227314603196027, "samples": [ 0.2518543777987361, 0.22819991409778595, 0.2207832643762231 ] }, "ttft_s": { "n": 3, "mean": 0.058666288231809936, "std": 0.013259953752573134, "samples": [ 0.07333162473514676, 0.05514443386346102, 0.04752280609682202 ] }, "actual_output_tokens": [ 33, 33, 33 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.16972966880227128, "std": 0.006189618702843324, "samples": [ 0.17565335799008608, 0.17023103218525648, 0.1633046162314713 ] }, "ttft_s": { "n": 3, "mean": 0.05399713292717934, "std": 0.00624194696431706, "samples": [ 0.059966676868498325, 0.05451026186347008, 0.04751446004956961 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.18053195594499508, "std": 0.00650786843288448, "samples": [ 0.1867452752776444, 0.18108567176386714, 0.17376492079347372 ] }, "ttft_s": { "n": 3, "mean": 0.054352966137230396, "std": 0.006426498216824092, "samples": [ 0.060608576983213425, 0.05468209274113178, 0.04776822868734598 ] }, "actual_output_tokens": [ 24, 24, 24 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.152838379299889, "std": 0.0010326853413987237, "samples": [ 0.152436182834208, 0.15401164907962084, 0.15206730598583817 ] }, "ttft_s": { "n": 3, "mean": 0.047985481563955545, "std": 0.0010006312890303353, "samples": [ 0.04764969274401665, 0.04911081911996007, 0.04719593282788992 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 15932.0, "sampled_peak_conversion_tree_rss_bytes": null, "file_size_is_not_runtime_memory": true } }, { "type": "Q3_K_M", "artifact": { "filename": "occamy-1.0-Q3_K_M.gguf", "bytes": 16764764160, "sha256": "a41137554d3d58e56dc2c043650ff92dbaf12822eb8be4a7a0d05ee1e9717d0c" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": true, "new_quantization_tensor_count": 733, "publication_gate": "corrected_copy_verified_see_release_json" }, "measurement_cohort": "initial_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 5998.353619, "std_tokens_per_second": 807.622891 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 232.862238, "std_tokens_per_second": 2.802633 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 5773.109934, "std_tokens_per_second": 897.221781 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 232.015999, "std_tokens_per_second": 3.481112 } }, "ppl": { "ppl": 6.740173, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.5016730000000003, "dump_base_ppl": 6.232933, "paired_ppl_delta": 0.50724, "mean_kl": 0.11943, "same_top_percent": 86.115, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 21, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 22, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 0, "base_passed": 22, "plus_passed": 22, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 0 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.2563619092106819, "std": 0.021052974073824043, "samples": [ 0.2616900550201535, 0.2742389137856662, 0.233156758826226 ] }, "ttft_s": { "n": 3, "mean": 0.06968650314956903, "std": 0.020314208797228114, "samples": [ 0.07448101369664073, 0.08717458369210362, 0.04740391205996275 ] }, "actual_output_tokens": [ 39, 39, 39 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.2115073922711114, "std": 0.006295930521400814, "samples": [ 0.21704057324677706, 0.21282450994476676, 0.2046570936217904 ] }, "ttft_s": { "n": 3, "mean": 0.054443600587546825, "std": 0.006302360700017953, "samples": [ 0.05996254226192832, 0.055792135652154684, 0.04757612384855747 ] }, "actual_output_tokens": [ 33, 33, 33 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.20666205557063222, "std": 0.005896988177183897, "samples": [ 0.2123115500435233, 0.20712924981489778, 0.20054536685347557 ] }, "ttft_s": { "n": 3, "mean": 0.05402073652173082, "std": 0.006035382932927082, "samples": [ 0.05999939516186714, 0.05413262918591499, 0.047930185217410326 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1585510945878923, "std": 0.006215774839613126, "samples": [ 0.16439999220892787, 0.1592292357236147, 0.15202405583113432 ] }, "ttft_s": { "n": 3, "mean": 0.053471331329395376, "std": 0.0061969726866420455, "samples": [ 0.05939495423808694, 0.053985938895493746, 0.047033100854605436 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.13979922886937857, "std": 0.006068988364186881, "samples": [ 0.14581927098333836, 0.1398959648795426, 0.13368245074525476 ] }, "ttft_s": { "n": 3, "mean": 0.053526426355044045, "std": 0.0061080691979616035, "samples": [ 0.05962987709790468, 0.053535652812570333, 0.047413749154657125 ] }, "actual_output_tokens": [ 18, 18, 18 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.14381595173229775, "std": 0.0009583601548351085, "samples": [ 0.1427937662228942, 0.14469420490786433, 0.1439598840661347 ] }, "ttft_s": { "n": 3, "mean": 0.04756571973363558, "std": 0.00036463602550216564, "samples": [ 0.04722997127100825, 0.04795362614095211, 0.04751356178894639 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 17196.0, "sampled_peak_conversion_tree_rss_bytes": null, "file_size_is_not_runtime_memory": true } }, { "type": "IQ4_XS", "artifact": { "filename": "occamy-1.0-IQ4_XS.gguf", "bytes": 18728777728, "sha256": "c37131d9049f031f209f2190bccb482a1d75ad9f2699ef86b67d1a193dbf1eae" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": true, "new_quantization_tensor_count": 733, "publication_gate": "corrected_copy_verified_see_release_json" }, "measurement_cohort": "initial_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 7037.354404, "std_tokens_per_second": 984.592869 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 227.522137, "std_tokens_per_second": 10.091295 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 7141.300973, "std_tokens_per_second": 228.242693 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 228.189154, "std_tokens_per_second": 6.502732 } }, "ppl": { "ppl": 6.31471, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.07620999999999967, "dump_base_ppl": 6.232933, "paired_ppl_delta": 0.081777, "mean_kl": 0.047864, "same_top_percent": 91.176, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 19, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 23, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 0, "base_passed": 23, "plus_passed": 23, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 1 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.24785351644580564, "std": 0.022533006700793713, "samples": [ 0.25092874839901924, 0.2686909670010209, 0.22394083393737674 ] }, "ttft_s": { "n": 3, "mean": 0.06964734274273117, "std": 0.0216137923393674, "samples": [ 0.0727028432302177, 0.08957079192623496, 0.046668393071740866 ] }, "actual_output_tokens": [ 37, 37, 37 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.20216554046298066, "std": 0.0060678831557134764, "samples": [ 0.20710565568879247, 0.20399846462532878, 0.19539250107482076 ] }, "ttft_s": { "n": 3, "mean": 0.0533928998435537, "std": 0.006068543636781278, "samples": [ 0.05854707304388285, 0.05492715071886778, 0.04670447576791048 ] }, "actual_output_tokens": [ 31, 31, 31 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.20751334416369596, "std": 0.00593079165783203, "samples": [ 0.21219580061733723, 0.20949996076524258, 0.2008442711085081 ] }, "ttft_s": { "n": 3, "mean": 0.05386460541437069, "std": 0.005826178248161945, "samples": [ 0.05866871401667595, 0.05554112419486046, 0.04738397803157568 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.15880161896348, "std": 0.00591694626914985, "samples": [ 0.16476176399737597, 0.15871425392106175, 0.15292883897200227 ] }, "ttft_s": { "n": 3, "mean": 0.05297360196709633, "std": 0.005740685834969104, "samples": [ 0.05869632074609399, 0.05300936894491315, 0.04721511621028185 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.14034538576379418, "std": 0.006114715819218857, "samples": [ 0.14566692104563117, 0.14170341100543737, 0.133665825240314 ] }, "ttft_s": { "n": 3, "mean": 0.053599674720317125, "std": 0.0061441134767715, "samples": [ 0.05892543215304613, 0.05499602761119604, 0.046877564396709204 ] }, "actual_output_tokens": [ 18, 18, 18 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.14290795071671406, "std": 0.0001312851205037561, "samples": [ 0.14293651515617967, 0.14302260195836425, 0.14276473503559828 ] }, "ttft_s": { "n": 3, "mean": 0.04667325410991907, "std": 0.00010770642693487369, "samples": [ 0.0467379679903388, 0.04673287412151694, 0.04654892021790147 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 19018.0, "sampled_peak_conversion_tree_rss_bytes": null, "file_size_is_not_runtime_memory": true } }, { "type": "Q4_K_M", "artifact": { "filename": "occamy-1.0-Q4_K_M.gguf", "bytes": 21166757696, "sha256": "ffb25f763ff9c27f5f4e2adcdef399c5654f9f840fdac33dffe7035ba8266a87" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": null, "new_quantization_tensor_count": null, "publication_gate": "unchanged_existing_file_not_part_of_this_publication" }, "measurement_cohort": "initial_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6346.994863, "std_tokens_per_second": 1065.664401 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 241.551554, "std_tokens_per_second": 3.040561 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6311.662919, "std_tokens_per_second": 552.284512 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 238.521534, "std_tokens_per_second": 7.650338 } }, "ppl": { "ppl": 6.242854, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.004354000000000191, "dump_base_ppl": 6.232933, "paired_ppl_delta": 0.009921, "mean_kl": 0.066201, "same_top_percent": 89.387, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 19, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 24, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 0, "base_passed": 24, "plus_passed": 24, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 1 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.24038906022906303, "std": 0.011171173105804654, "samples": [ 0.24505678098648787, 0.2484693438746035, 0.22764105582609773 ] }, "ttft_s": { "n": 3, "mean": 0.06878846759597461, "std": 0.010579074468882262, "samples": [ 0.07350242510437965, 0.07619113894179463, 0.056671838741749525 ] }, "actual_output_tokens": [ 37, 37, 37 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.16141746751964092, "std": 0.0069305576254149945, "samples": [ 0.16763887973502278, 0.1626660218462348, 0.15394750097766519 ] }, "ttft_s": { "n": 3, "mean": 0.0544493809963266, "std": 0.00655935901626448, "samples": [ 0.06016011582687497, 0.0559028210118413, 0.04728520615026355 ] }, "actual_output_tokens": [ 23, 23, 23 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.20411067781969905, "std": 0.0038701337013363757, "samples": [ 0.20696542598307133, 0.20566084375604987, 0.19970576371997595 ] }, "ttft_s": { "n": 3, "mean": 0.05501478211954236, "std": 0.0049201494388034055, "samples": [ 0.059163028839975595, 0.05630248971283436, 0.04957882780581713 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.15785645057136813, "std": 0.006728403196999599, "samples": [ 0.1640180773101747, 0.15887407306581736, 0.15067720133811235 ] }, "ttft_s": { "n": 3, "mean": 0.054961634489397206, "std": 0.006323736960668102, "samples": [ 0.06073369225487113, 0.05594886792823672, 0.04820234328508377 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1396949328482151, "std": 0.006702612691552292, "samples": [ 0.14523081574589014, 0.14161106199026108, 0.1322429208084941 ] }, "ttft_s": { "n": 3, "mean": 0.05537131801247597, "std": 0.006674064226008372, "samples": [ 0.06098819896578789, 0.05713246390223503, 0.047993291169404984 ] }, "actual_output_tokens": [ 18, 18, 18 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.14108201302587986, "std": 0.00027350336194442785, "samples": [ 0.14139112597331405, 0.1408714121207595, 0.14098350098356605 ] }, "ttft_s": { "n": 3, "mean": 0.04800532308096687, "std": 0.00021959045286995215, "samples": [ 0.04813843499869108, 0.047751869075000286, 0.04812566516920924 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 21916.0, "sampled_peak_conversion_tree_rss_bytes": null, "file_size_is_not_runtime_memory": true } }, { "type": "Q5_K_M", "artifact": { "filename": "occamy-1.0-Q5_K_M.gguf", "bytes": 24729131008, "sha256": "43cb6af4a64e846a32b3e87502d51008a9a8123bd76f9462120a6dff7ae6c6e4" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": true, "new_quantization_tensor_count": 733, "publication_gate": "corrected_copy_verified_see_release_json" }, "measurement_cohort": "initial_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6388.107699, "std_tokens_per_second": 711.301751 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 237.844422, "std_tokens_per_second": 9.98583 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6310.46296, "std_tokens_per_second": 326.159437 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 238.311167, "std_tokens_per_second": 9.629646 } }, "ppl": { "ppl": 6.248758, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.010257999999999434, "dump_base_ppl": 6.232933, "paired_ppl_delta": 0.015826, "mean_kl": 0.025792, "same_top_percent": 93.627, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 19, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 23, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 1, "base_passed": 23, "plus_passed": 23, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 1 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.23685449874028563, "std": 0.016028458541619047, "samples": [ 0.24549263576045632, 0.2467110580764711, 0.2183598023839295 ] }, "ttft_s": { "n": 3, "mean": 0.06554047126943867, "std": 0.015546242471568189, "samples": [ 0.07433594577014446, 0.07469505397602916, 0.04759041406214237 ] }, "actual_output_tokens": [ 37, 37, 37 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1836497032394012, "std": 0.006403216372814672, "samples": [ 0.18930969666689634, 0.1849397229962051, 0.1766996900551021 ] }, "ttft_s": { "n": 3, "mean": 0.054122584871947765, "std": 0.0064213559550823284, "samples": [ 0.05992884561419487, 0.05521311378106475, 0.04722579522058368 ] }, "actual_output_tokens": [ 28, 28, 28 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.20173018674055734, "std": 0.005786127303538893, "samples": [ 0.20760195795446634, 0.20155491633340716, 0.19603368593379855 ] }, "ttft_s": { "n": 3, "mean": 0.05395927963157495, "std": 0.005969194462955443, "samples": [ 0.05992028955370188, 0.05397561518475413, 0.047981934156268835 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1573992989336451, "std": 0.006793295644873734, "samples": [ 0.16212646570056677, 0.16045690421015024, 0.14961452689021826 ] }, "ttft_s": { "n": 3, "mean": 0.053940510687728725, "std": 0.006182955986684925, "samples": [ 0.060089601669460535, 0.0540076931938529, 0.04772423719987273 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.13717101405685148, "std": 0.00665833142713536, "samples": [ 0.14394385507330298, 0.1369357593357563, 0.1306334277614951 ] }, "ttft_s": { "n": 3, "mean": 0.05355409005035957, "std": 0.006591863368307163, "samples": [ 0.060212945099920034, 0.05341799929738045, 0.04703132575377822 ] }, "actual_output_tokens": [ 18, 18, 18 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.14026402433713278, "std": 0.0006408657264824784, "samples": [ 0.14099251525476575, 0.14001240488141775, 0.13978715287521482 ] }, "ttft_s": { "n": 3, "mean": 0.04764915940662225, "std": 0.0006587129685173037, "samples": [ 0.04840056225657463, 0.04737567389383912, 0.047171242069453 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 24664.0, "sampled_peak_conversion_tree_rss_bytes": null, "file_size_is_not_runtime_memory": true } }, { "type": "Q6_K", "artifact": { "filename": "occamy-1.0-Q6_K.gguf", "bytes": 28514152448, "sha256": "43e1b751fe3ebc23bccb72c3183763aabba33741854264033f4e7bb56865c829" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": true, "new_quantization_tensor_count": 733, "publication_gate": "corrected_copy_verified_see_release_json" }, "measurement_cohort": "initial_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 5759.135102, "std_tokens_per_second": 824.856082 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 223.943892, "std_tokens_per_second": 4.211126 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 5765.39998, "std_tokens_per_second": 418.057023 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 221.34997, "std_tokens_per_second": 9.847091 } }, "ppl": { "ppl": 6.259335, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.020834999999999937, "dump_base_ppl": 6.232933, "paired_ppl_delta": 0.026402, "mean_kl": 0.015831, "same_top_percent": 94.792, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 18, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 24, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 0, "base_passed": 24, "plus_passed": 24, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 1 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.24852733950441083, "std": 0.017025041119127823, "samples": [ 0.2598795732483268, 0.25675075920298696, 0.22895168606191874 ] }, "ttft_s": { "n": 3, "mean": 0.06656573728347819, "std": 0.016371381907610793, "samples": [ 0.07821711525321007, 0.07363214623183012, 0.047847950365394354 ] }, "actual_output_tokens": [ 37, 37, 37 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1979097768974801, "std": 0.005825300530141866, "samples": [ 0.20308292424306273, 0.19904651306569576, 0.19159989338368177 ] }, "ttft_s": { "n": 3, "mean": 0.055153677705675364, "std": 0.006026722663691082, "samples": [ 0.06085830321535468, 0.05575307784602046, 0.04884965205565095 ] }, "actual_output_tokens": [ 29, 29, 29 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.21267080632969737, "std": 0.006922754916511469, "samples": [ 0.21936996094882488, 0.21309820283204317, 0.20554425520822406 ] }, "ttft_s": { "n": 3, "mean": 0.05530608491972089, "std": 0.0066810864073461144, "samples": [ 0.061676736921072006, 0.055888783652335405, 0.04835273418575525 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.16499523368353644, "std": 0.008838159406532998, "samples": [ 0.1741663608700037, 0.1642866381444037, 0.15653270203620195 ] }, "ttft_s": { "n": 3, "mean": 0.05495617507646481, "std": 0.006580358907119822, "samples": [ 0.061120184138417244, 0.055721879936754704, 0.04802646115422249 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.14352990267798305, "std": 0.006660642715633453, "samples": [ 0.15003244299441576, 0.14383558183908463, 0.13672168320044875 ] }, "ttft_s": { "n": 3, "mean": 0.054734770519038044, "std": 0.006621428598726721, "samples": [ 0.061189260333776474, 0.05505689000710845, 0.0479581612162292 ] }, "actual_output_tokens": [ 18, 18, 18 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.14678852120414376, "std": 0.00044233899772274494, "samples": [ 0.1464279848150909, 0.14728211658075452, 0.14665546221658587 ] }, "ttft_s": { "n": 3, "mean": 0.04816233723734816, "std": 0.00034281975884013777, "samples": [ 0.047927014995366335, 0.04855566658079624, 0.0480043301358819 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 28210.0, "sampled_peak_conversion_tree_rss_bytes": null, "file_size_is_not_runtime_memory": true } }, { "type": "Q8_0", "artifact": { "filename": "occamy-1.0-Q8_0.gguf", "bytes": 36903139456, "sha256": "9bd6eceae8434098c8c7ebec33e4ca0d0bd3f004c2bddf37d16cd906448fdbdd" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": null, "new_quantization_tensor_count": null, "publication_gate": "unchanged_existing_file_not_part_of_this_publication" }, "measurement_cohort": "initial_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6719.326314, "std_tokens_per_second": 1768.730104 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 223.50401, "std_tokens_per_second": 4.416177 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 7072.92687, "std_tokens_per_second": 340.434418 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 223.213252, "std_tokens_per_second": 4.714636 } }, "ppl": { "ppl": 6.257434, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.018933999999999784, "dump_base_ppl": 6.232933, "paired_ppl_delta": 0.024501, "mean_kl": 0.012633, "same_top_percent": 95.453, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 18, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 23, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 1, "base_passed": 23, "plus_passed": 23, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 2 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.2459441895286242, "std": 0.014663235738846155, "samples": [ 0.25649824598804116, 0.25213317200541496, 0.22920115059241652 ] }, "ttft_s": { "n": 3, "mean": 0.06376485604171951, "std": 0.014135603953686677, "samples": [ 0.07462143991142511, 0.06889195321127772, 0.04778117500245571 ] }, "actual_output_tokens": [ 37, 37, 37 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.18794798214609423, "std": 0.006179040525984396, "samples": [ 0.19346647569909692, 0.1891053938306868, 0.181272076908499 ] }, "ttft_s": { "n": 3, "mean": 0.05491731191674868, "std": 0.006045734620657219, "samples": [ 0.06032932084053755, 0.05603014584630728, 0.04839246906340122 ] }, "actual_output_tokens": [ 27, 27, 27 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.21181433197731772, "std": 0.006002262869676362, "samples": [ 0.2177629149518907, 0.2119202888570726, 0.2057597921229899 ] }, "ttft_s": { "n": 3, "mean": 0.0546454853999118, "std": 0.005965737947834603, "samples": [ 0.060577703174203634, 0.054711970034986734, 0.048646782990545034 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1627510185353458, "std": 0.005530989530811421, "samples": [ 0.1680593630298972, 0.1631722250021994, 0.15702146757394075 ] }, "ttft_s": { "n": 3, "mean": 0.05425141534457604, "std": 0.005587700042181537, "samples": [ 0.059780027251690626, 0.05436777416616678, 0.048606444615870714 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1728828949853778, "std": 0.00605541141079351, "samples": [ 0.17855435889214277, 0.1735888971015811, 0.1665054289624095 ] }, "ttft_s": { "n": 3, "mean": 0.05464269003520409, "std": 0.00599772028006368, "samples": [ 0.06033726129680872, 0.05520883994176984, 0.04838196886703372 ] }, "actual_output_tokens": [ 24, 24, 24 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.14644432999193668, "std": 0.0005001441619092046, "samples": [ 0.14639546303078532, 0.14597041299566627, 0.14696711394935846 ] }, "ttft_s": { "n": 3, "mean": 0.047828994070490204, "std": 0.0004616145234633238, "samples": [ 0.047815748024731874, 0.04737414512783289, 0.04829708905890584 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 36432.0, "sampled_peak_conversion_tree_rss_bytes": null, "file_size_is_not_runtime_memory": true } }, { "type": "IQ3_XS", "artifact": { "filename": "occamy-1.0-IQ3_XS.gguf", "bytes": 14484144128, "sha256": "126b5265fdb52dc53b69be5535c436267cd83acdc87f1820a02ade728d3f5ef4" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": true, "new_quantization_tensor_count": 733, "publication_gate": "corrected_copy_verified_see_release_json" }, "measurement_cohort": "parallel_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6166.288001, "std_tokens_per_second": 1164.815915 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 203.790601, "std_tokens_per_second": 2.581116 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6518.855174, "std_tokens_per_second": 339.58132 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 203.767644, "std_tokens_per_second": 3.585604 } }, "ppl": { "ppl": 6.597194, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.35869399999999985, "dump_base_ppl": 6.232933, "paired_ppl_delta": 0.364261, "mean_kl": 0.134423, "same_top_percent": 85.025, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 19, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 24, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 1, "base_passed": 24, "plus_passed": 24, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 3 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.3279776278262337, "std": 0.025276325665750526, "samples": [ 0.33012155815958977, 0.35211370326578617, 0.3016976220533252 ] }, "ttft_s": { "n": 3, "mean": 0.0735549380381902, "std": 0.024266229164964494, "samples": [ 0.07595714088529348, 0.09653072524815798, 0.048176947981119156 ] }, "actual_output_tokens": [ 48, 48, 48 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.21508905695130429, "std": 0.005795080106844155, "samples": [ 0.220191759057343, 0.216286676004529, 0.20878873579204082 ] }, "ttft_s": { "n": 3, "mean": 0.05587405587236086, "std": 0.0060829912063510684, "samples": [ 0.061314751859754324, 0.057001031935214996, 0.049306383822113276 ] }, "actual_output_tokens": [ 30, 30, 30 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.22494482280065617, "std": 0.007589200306552562, "samples": [ 0.2328932760283351, 0.2241663010790944, 0.21777489129453897 ] }, "ttft_s": { "n": 3, "mean": 0.05563883235057195, "std": 0.007136297036218071, "samples": [ 0.06305283913388848, 0.05504648992791772, 0.04881716798990965 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.17111475750183067, "std": 0.006554212135615407, "samples": [ 0.17731431685388088, 0.1717742057517171, 0.164255749899894 ] }, "ttft_s": { "n": 3, "mean": 0.055112294697513185, "std": 0.006533093013387528, "samples": [ 0.06127970293164253, 0.055790719110518694, 0.04826646205037832 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.15068098731959859, "std": 0.006252782206673077, "samples": [ 0.156892875675112, 0.15076198801398277, 0.144388098269701 ] }, "ttft_s": { "n": 3, "mean": 0.055363136964539685, "std": 0.006296196070956357, "samples": [ 0.06161465495824814, 0.05545156169682741, 0.04902319423854351 ] }, "actual_output_tokens": [ 18, 18, 18 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.15474400902166963, "std": 0.0005149841880116399, "samples": [ 0.15432095481082797, 0.15531744435429573, 0.15459362789988518 ] }, "ttft_s": { "n": 3, "mean": 0.04884775805597504, "std": 0.00046250777117193677, "samples": [ 0.04848006879910827, 0.04936703806743026, 0.048696167301386595 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 15020.0, "sampled_peak_conversion_tree_rss_bytes": 3942006784, "file_size_is_not_runtime_memory": true } }, { "type": "IQ4_NL", "artifact": { "filename": "occamy-1.0-IQ4_NL.gguf", "bytes": 19779278848, "sha256": "6dcf068e182a482b0bdbc493ba7154712a034825d47ebf348d9dee7bece007c0" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": true, "new_quantization_tensor_count": 733, "publication_gate": "corrected_copy_verified_see_release_json" }, "measurement_cohort": "parallel_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6856.118179, "std_tokens_per_second": 1322.337721 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 241.737765, "std_tokens_per_second": 12.595675 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6910.829549, "std_tokens_per_second": 666.879687 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 244.747027, "std_tokens_per_second": 3.578141 } }, "ppl": { "ppl": 6.318163, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.07966300000000004, "dump_base_ppl": 6.232933, "paired_ppl_delta": 0.08523, "mean_kl": 0.046354, "same_top_percent": 91.238, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 19, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 23, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 1, "base_passed": 23, "plus_passed": 23, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 3 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.23681716822708646, "std": 0.019683988387632887, "samples": [ 0.24371080193668604, 0.2521271570585668, 0.21461354568600655 ] }, "ttft_s": { "n": 3, "mean": 0.06833163400491078, "std": 0.019009738096268068, "samples": [ 0.07510404428467155, 0.08302776794880629, 0.04686308978125453 ] }, "actual_output_tokens": [ 37, 37, 37 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.17535486879448095, "std": 0.005153032552155704, "samples": [ 0.17990116402506828, 0.17640619771555066, 0.16975724464282393 ] }, "ttft_s": { "n": 3, "mean": 0.05686028953641653, "std": 0.005137851382395989, "samples": [ 0.061482592951506376, 0.05776989087462425, 0.05132838478311896 ] }, "actual_output_tokens": [ 26, 26, 26 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.20158252958208323, "std": 0.007303588719258058, "samples": [ 0.2088620811700821, 0.2016303688287735, 0.19425513874739408 ] }, "ttft_s": { "n": 3, "mean": 0.056268551697333656, "std": 0.006735278909863007, "samples": [ 0.06282382505014539, 0.05661517707630992, 0.049366652965545654 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.15565358443806568, "std": 0.005998443204966175, "samples": [ 0.1613878458738327, 0.15615097526460886, 0.1494219321757555 ] }, "ttft_s": { "n": 3, "mean": 0.055379202434172235, "std": 0.006029209991806074, "samples": [ 0.06116515304893255, 0.0558393532410264, 0.049133101012557745 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.13901671037698785, "std": 0.006444154475504606, "samples": [ 0.14267711620777845, 0.14279705099761486, 0.13157596392557025 ] }, "ttft_s": { "n": 3, "mean": 0.05499925510957837, "std": 0.0057071932300593025, "samples": [ 0.06045859307050705, 0.055466268211603165, 0.0490729040466249 ] }, "actual_output_tokens": [ 18, 18, 18 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.14053894253447652, "std": 0.0003433460067537544, "samples": [ 0.1409282316453755, 0.14040931314229965, 0.1402792828157544 ] }, "ttft_s": { "n": 3, "mean": 0.04910820738101999, "std": 0.00029116060117625943, "samples": [ 0.04940951196476817, 0.04908672999590635, 0.048828380182385445 ] }, "actual_output_tokens": [ 20, 20, 20 ] } }, "memory": { "sampled_peak_device_used_mib": 20004.0, "sampled_peak_conversion_tree_rss_bytes": 3941294080, "file_size_is_not_runtime_memory": true } }, { "type": "Q4_K_S", "artifact": { "filename": "occamy-1.0-Q4_K_S.gguf", "bytes": 19889903616, "sha256": "d7b4dc1b3f8b3be6fa0a16eda8bfb613119afa5a7c8db49d823b59b8af988b0d" }, "validation": { "load_tested": true, "benchmarked": true, "new_quantization_full_finite_scan_passed": true, "new_quantization_tensor_count": 733, "publication_gate": "corrected_copy_verified_see_release_json" }, "measurement_cohort": "parallel_single_gpu_jobs", "engine": { "pp512": { "kind": "pp", "prompt_tokens": 512, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6305.7643, "std_tokens_per_second": 1593.437097 }, "tg128_d512": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 512, "mean_tokens_per_second": 239.666664, "std_tokens_per_second": 4.069765 }, "pp2048": { "kind": "pp", "prompt_tokens": 2048, "generated_tokens": 0, "decode_initial_context_tokens": null, "mean_tokens_per_second": 6584.668241, "std_tokens_per_second": 270.456255 }, "tg128_d2048": { "kind": "tg", "prompt_tokens": 0, "generated_tokens": 128, "decode_initial_context_tokens": 2048, "mean_tokens_per_second": 237.581923, "std_tokens_per_second": 6.619769 } }, "ppl": { "ppl": 6.282276, "direct_bf16_reference_ppl": 6.2385, "ppl_delta_vs_direct_bf16": 0.04377600000000026, "dump_base_ppl": 6.232933, "paired_ppl_delta": 0.049343, "mean_kl": 0.051069, "same_top_percent": 90.6, "chunks": 32, "n_ctx": 512, "scored_tokens": 8160 }, "quality": { "math_strict": { "passed": 0, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 24 }, "math_flexible": { "passed": 21, "scored": 24, "total_planned": 24, "pending_or_missing": null, "truncated": null, "base_passed": null, "plus_passed": null, "extraction_invalid": 0 }, "code": { "passed": 23, "scored": 24, "total_planned": 24, "pending_or_missing": 0, "truncated": 0, "base_passed": 23, "plus_passed": 23, "extraction_invalid": null }, "code_outputs_actually_executed": 24, "json": { "passed": 4, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "tool": { "passed": 6, "scored": 6, "planned": 6, "pending": 0, "truncated": 0 }, "request_errors": 0, "truncations": 0 }, "e2e": { "text_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.4898868848880132, "std": 0.016192211295917417, "samples": [ 0.5000728080049157, 0.49837232287973166, 0.47121552377939224 ] }, "ttft_s": { "n": 3, "mean": 0.06592472270131111, "std": 0.01537102868090097, "samples": [ 0.07626282703131437, 0.07325011817738414, 0.04826122289523482 ] }, "actual_output_tokens": [ 92, 92, 92 ] }, "text_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.17950111270571747, "std": 0.006368340839696904, "samples": [ 0.1853360990062356, 0.18045924417674541, 0.17270799493417144 ] }, "ttft_s": { "n": 3, "mean": 0.054418385649720825, "std": 0.0062758226975835945, "samples": [ 0.06013060500845313, 0.05542412307113409, 0.04770042886957526 ] }, "actual_output_tokens": [ 27, 27, 27 ] }, "code_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.20182643691077828, "std": 0.006497579709304292, "samples": [ 0.20816185884177685, 0.20213943906128407, 0.19517801282927394 ] }, "ttft_s": { "n": 3, "mean": 0.053930392023175955, "std": 0.006146241344660085, "samples": [ 0.059900789987295866, 0.05426815012469888, 0.04762223595753312 ] }, "actual_output_tokens": [ 32, 32, 32 ] }, "code_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1552351638674736, "std": 0.005967008568245169, "samples": [ 0.1610678369179368, 0.15549532463774085, 0.14914233004674315 ] }, "ttft_s": { "n": 3, "mean": 0.05341656723370155, "std": 0.0060258322134452424, "samples": [ 0.059284294955432415, 0.05372121883556247, 0.04724418791010976 ] }, "actual_output_tokens": [ 22, 22, 22 ] }, "json_0": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.16569887846708298, "std": 0.005754651722380783, "samples": [ 0.17087450018152595, 0.16672018403187394, 0.15950195118784904 ] }, "ttft_s": { "n": 3, "mean": 0.0542328303369383, "std": 0.005841838087165158, "samples": [ 0.05981520004570484, 0.05472111701965332, 0.04816217394545674 ] }, "actual_output_tokens": [ 24, 24, 24 ] }, "json_1": { "requests": 3, "errors": 0, "truncated": 0, "latency_s": { "n": 3, "mean": 0.1732787311387559, "std": 0.00031225285517904, "samples": [ 0.17363912938162684, 0.17310784198343754, 0.17308922205120325 ] }, "ttft_s": { "n": 3, "mean": 0.04807448045661052, "std": 0.00024927805570199686, "samples": [ 0.04830955620855093, 0.04781308816745877, 0.04810079699382186 ] }, "actual_output_tokens": [ 27, 27, 27 ] } }, "memory": { "sampled_peak_device_used_mib": 20110.0, "sampled_peak_conversion_tree_rss_bytes": 3940864000, "file_size_is_not_runtime_memory": true } } ], "release_artifacts": "release.json", "benchmark_artifact_hashes_are_pre_correction": true }