| { |
| "schema_version": 1, |
| "generated_utc": "2026-09-17T23:56:57.083966+00:00", |
| "scope": "BF16 reference and twelve concrete GGUF quantization artifacts; small fixed screening subsets", |
| "source": { |
| "repo": "Accio-Lab/occamy-1.0", |
| "revision": "8f8e0e58a3c9df042be1a3fa2c191fd8047acfb8" |
| }, |
| "runtime": { |
| "engine": "llama.cpp", |
| "commit": "972d2313bc0bf0a45f634f77d95c9fb03aeab12c", |
| "hardware": "NVIDIA B200", |
| "gpus_per_job": 1, |
| "mtp": false, |
| "cpu_threads_per_job": 8, |
| "flash_attention": true, |
| "kv_cache_type": "f16", |
| "batch": 2048, |
| "microbatch": 512, |
| "engine_repetitions": 5, |
| "engine_warmup": true |
| }, |
| "tokenizer_compatibility": { |
| "benchmark_effective_pretokenizer": "qwen35", |
| "comparison_reference": "GGUF BF16", |
| "hf_source_behavior": "NFC normalization plus qwen2 tokenization", |
| "hf_source_equivalence": "Mismatch observed on small probes; these benchmarks do not establish equivalence to the HF source tokenizer", |
| "publication_gate": "ten_corrected_copies_verified_see_release_json; Q4_K_M/Q8_0 unchanged; BF16 reference only" |
| }, |
| "quality_method": { |
| "cases_per_model": 60, |
| "math_cases": 24, |
| "code_cases": 24, |
| "json_cases": 6, |
| "tool_cases": 6, |
| "temperature": 0, |
| "seed": 20260918, |
| "max_generated_tokens": 512, |
| "thinking": false, |
| "retries": 0, |
| "per_model_concurrency": 1, |
| "selection_algorithm": "sort SHA256(\"20260918:\" + stable ID), take first 24 for each math/code suite", |
| "frozen_data_sha256": "8c587abf50aa5b06b379ff83070f0c56e9eb889dd1b1073a9fb3834677888f63", |
| "gsm8k_source_revision": "3101c7d5072418e28b9008a6636bde82a006892c", |
| "humanevalplus_version": "v0.1.10", |
| "lm_eval_revision": "d6de81643928d653435c431bae19945d41d32520", |
| "evalplus_revision": "26d6d00bb1fd0fa37f39c99d5290da67891d1c5e", |
| "code_scoring": "Pinned EvalPlus sanitizer and base + extended tests; actual execution in an isolated, network-disabled, resource-limited CPU sandbox", |
| "strict_json": "Parse the complete response without stripping Markdown fences", |
| "tool_scoring": "Expected native tool name and argument structure; first tool call only" |
| }, |
| "ppl_method": { |
| "dataset": "Salesforce/wikitext", |
| "configuration": "wikitext-2-raw-v1", |
| "split": "test", |
| "revision": "b08601e04326c79dfdd32d625aee71d232d685c3", |
| "chunks": 32, |
| "context_tokens": 512, |
| "actual_scored_tokens": 8160, |
| "direct_bf16_ppl": 6.2385, |
| "reference_dump": "lossy uint16 log-probabilities, not BF16 tensors", |
| "bf16_self_replay_ppl": 6.23851, |
| "decoded_dump_base_ppl": 6.232933, |
| "dump_floor_delta": 0.005577 |
| }, |
| "limitations": [ |
| "These are fixed small subsets, not full official GSM8K/HumanEval+ results; small score differences do not establish a ranking.", |
| "The math prompt did not ask for ####, so strict extraction failure is not synonymous with wrong arithmetic. Strict and upstream flexible scores are separate.", |
| "Truncated outputs remain in denominators. A passed inference job does not imply a correct answer.", |
| "Engine throughput excludes tokenization and sampling. End-to-end tests use six requests repeated three times with cache disabled.", |
| "Three additional artifacts used concurrent single-GPU jobs across four cards. Physical-card variation and concurrent CPU/IO load confound comparison with the initial single-card runs.", |
| "Ten newly generated types share the same BF16 input and imatrix. Reused Q4_K_M/Q8_0 used different historical conversion recipes; this is not a bit-width-only controlled experiment.", |
| "Benchmark tokenization uses qwen35 to align with GGUF BF16. Source-tokenizer probes found NFC plus qwen2 behavior and disagreement on small inputs. GGUF-reference alignment does not establish HF-source equivalence. Reused Q4/Q8 runtime overrides do not modify artifact bytes; release copies have corrected metadata and bounded validation recorded in release.json; this original benchmark protocol is unchanged.", |
| "Lower-bit PPL degradation is a quantization quality tradeoff, not detected file damage: newly generated artifacts passed full finite scans and load tests. These checks do not establish BF16-equivalent quality.", |
| "Direct BF16 PPL is rounded to four decimals. Native dump-relative PPL/KL metrics use a lossy reference and retain their numerical floor.", |
| "Memory values are sampled device/process-tree peaks and lower bounds; file size is not memory usage." |
| ], |
| "models": [ |
| { |
| "type": "BF16", |
| "artifact": { |
| "filename": "occamy-1.0-BF16.gguf", |
| "bytes": 69376636992, |
| "sha256": "bbad6c5aaf28f20a700748613f998a2c0a98b6df94766ec37bf5c4a2cbf2aadf" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": null, |
| "new_quantization_tensor_count": null, |
| "publication_gate": "reference_only_not_for_publication" |
| }, |
| "measurement_cohort": "initial_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 2959.437099, |
| "std_tokens_per_second": 74.636743 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 192.962461, |
| "std_tokens_per_second": 3.481134 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 3009.479795, |
| "std_tokens_per_second": 17.279071 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 193.153558, |
| "std_tokens_per_second": 2.790821 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.2385, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.0, |
| "dump_base_ppl": null, |
| "paired_ppl_delta": null, |
| "mean_kl": null, |
| "same_top_percent": null, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 18, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 23, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 1, |
| "base_passed": 23, |
| "plus_passed": 23, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 2 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.297846841160208, |
| "std": 0.04942725094230669, |
| "samples": [ |
| 0.35285714128986, |
| 0.28351185796782374, |
| 0.2571715242229402 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.09064460825175047, |
| "std": 0.04932459176038486, |
| "samples": [ |
| 0.1459049042314291, |
| 0.07495766412466764, |
| 0.05107125639915466 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 37, |
| 37, |
| 37 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.219152867483596, |
| "std": 0.006254457556250989, |
| "samples": [ |
| 0.2243440099991858, |
| 0.22090554516762495, |
| 0.21220904728397727 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05816571911176046, |
| "std": 0.005823544185968752, |
| "samples": [ |
| 0.06289689987897873, |
| 0.059938439168035984, |
| 0.05166181828826666 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 29, |
| 29, |
| 29 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.23775445157662034, |
| "std": 0.011331934035808278, |
| "samples": [ |
| 0.25013370905071497, |
| 0.235236092004925, |
| 0.22789355367422104 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05843055512135228, |
| "std": 0.007894567590682097, |
| "samples": [ |
| 0.06648584641516209, |
| 0.05809863796457648, |
| 0.050707180984318256 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.18072297874217233, |
| "std": 0.00249112178796162, |
| "samples": [ |
| 0.18359363311901689, |
| 0.17912892112508416, |
| 0.17944638198241591 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05705882733066877, |
| "std": 0.005430367177984483, |
| "samples": [ |
| 0.06209668703377247, |
| 0.05777314119040966, |
| 0.05130665376782417 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.15852034262691936, |
| "std": 0.004939948320688301, |
| "samples": [ |
| 0.1629537669941783, |
| 0.15941195003688335, |
| 0.1531953108496964 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05882052223508557, |
| "std": 0.004834114778084178, |
| "samples": [ |
| 0.0630911709740758, |
| 0.05979804787784815, |
| 0.05357234785333276 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 18, |
| 18, |
| 18 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.16173411967853704, |
| "std": 0.0002234445576423375, |
| "samples": [ |
| 0.16180179407820106, |
| 0.16148466104641557, |
| 0.16191590391099453 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05093107822661599, |
| "std": 0.0002673094458067316, |
| "samples": [ |
| 0.05099930986762047, |
| 0.05063626589253545, |
| 0.05115765891969204 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 66626.0, |
| "sampled_peak_conversion_tree_rss_bytes": null, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "IQ2_M", |
| "artifact": { |
| "filename": "occamy-1.0-IQ2_M.gguf", |
| "bytes": 11659235328, |
| "sha256": "93a72b1723a06b5572d6fe9bafec42dfac1ef8b93cb1f0bf7389143b085ea8d8" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": true, |
| "new_quantization_tensor_count": 733, |
| "publication_gate": "corrected_copy_verified_see_release_json" |
| }, |
| "measurement_cohort": "initial_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 5495.642397, |
| "std_tokens_per_second": 980.55164 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 211.597565, |
| "std_tokens_per_second": 8.840749 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 5814.877035, |
| "std_tokens_per_second": 229.21048 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 204.737153, |
| "std_tokens_per_second": 20.239044 |
| } |
| }, |
| "ppl": { |
| "ppl": 7.444683, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 1.2061830000000002, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 1.21175, |
| "mean_kl": 0.294021, |
| "same_top_percent": 78.407, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 19, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 22, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 0, |
| "base_passed": 22, |
| "plus_passed": 22, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 2 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.37960403024529416, |
| "std": 0.0183496217303782, |
| "samples": [ |
| 0.38816892029717565, |
| 0.3921052264049649, |
| 0.35853794403374195 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06687216305484374, |
| "std": 0.016673994604842403, |
| "samples": [ |
| 0.07569061405956745, |
| 0.07728516031056643, |
| 0.047640714794397354 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 60, |
| 60, |
| 60 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.17920300100619593, |
| "std": 0.006660321287403449, |
| "samples": [ |
| 0.18529233383014798, |
| 0.18022649129852653, |
| 0.17209017788991332 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.054271044209599495, |
| "std": 0.006716145759241409, |
| "samples": [ |
| 0.06040927581489086, |
| 0.05530658410862088, |
| 0.04709727270528674 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 24, |
| 24, |
| 24 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.22057364167024693, |
| "std": 0.0076809238656142635, |
| "samples": [ |
| 0.22863844409585, |
| 0.219737458974123, |
| 0.21334502194076777 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05372514917204777, |
| "std": 0.006597803999765558, |
| "samples": [ |
| 0.06019252073019743, |
| 0.053978703916072845, |
| 0.04700422286987305 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.16828030332302055, |
| "std": 0.006290562022786465, |
| "samples": [ |
| 0.1743790772743523, |
| 0.16864776983857155, |
| 0.16181406285613775 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05391235618541638, |
| "std": 0.0063584671741067356, |
| "samples": [ |
| 0.06007832894101739, |
| 0.05428128084167838, |
| 0.04737745877355337 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.14788455019394556, |
| "std": 0.006549307990223488, |
| "samples": [ |
| 0.15445623407140374, |
| 0.14783956669270992, |
| 0.14135784981772304 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05407171711946527, |
| "std": 0.006633319572250986, |
| "samples": [ |
| 0.060716153122484684, |
| 0.05404942808672786, |
| 0.04744957014918327 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 18, |
| 18, |
| 18 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.15408787969499826, |
| "std": 0.0017580302518619073, |
| "samples": [ |
| 0.1521415263414383, |
| 0.1545616048388183, |
| 0.1555605079047382 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.04991562866295377, |
| "std": 0.0017963152097716486, |
| "samples": [ |
| 0.04790793592110276, |
| 0.050468225963413715, |
| 0.051370724104344845 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 12328.0, |
| "sampled_peak_conversion_tree_rss_bytes": null, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "Q2_K", |
| "artifact": { |
| "filename": "occamy-1.0-Q2_K.gguf", |
| "bytes": 12939593728, |
| "sha256": "caed9d813887780c3b614bec677b46e81acefdf8b13a0cfd6877df4b48ca79f4" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": true, |
| "new_quantization_tensor_count": 733, |
| "publication_gate": "corrected_copy_verified_see_release_json" |
| }, |
| "measurement_cohort": "initial_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 4806.397676, |
| "std_tokens_per_second": 994.100097 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 233.017661, |
| "std_tokens_per_second": 6.454193 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 5085.950076, |
| "std_tokens_per_second": 122.892266 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 226.998914, |
| "std_tokens_per_second": 18.288104 |
| } |
| }, |
| "ppl": { |
| "ppl": 7.418311, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 1.179811, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 1.185378, |
| "mean_kl": 0.266877, |
| "same_top_percent": 80.208, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 19, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 22, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 1, |
| "base_passed": 22, |
| "plus_passed": 22, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 3 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.2147925652874013, |
| "std": 0.014104193413298823, |
| "samples": [ |
| 0.22708952520042658, |
| 0.21789161767810583, |
| 0.19939655298367143 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06211134900028507, |
| "std": 0.013592860760596326, |
| "samples": [ |
| 0.07421206217259169, |
| 0.06471809465438128, |
| 0.047403890173882246 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.2009845453624924, |
| "std": 0.006619210547371885, |
| "samples": [ |
| 0.20724572893232107, |
| 0.20165028516203165, |
| 0.19405762199312449 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.053906473175932966, |
| "std": 0.006562911742503776, |
| "samples": [ |
| 0.05999909481033683, |
| 0.0547629538923502, |
| 0.046957370825111866 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 31, |
| 31, |
| 31 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.20494974901278815, |
| "std": 0.006414678876178007, |
| "samples": [ |
| 0.2108617490157485, |
| 0.20585824083536863, |
| 0.19812925718724728 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.053499534260481596, |
| "std": 0.006547396360666597, |
| "samples": [ |
| 0.059484392404556274, |
| 0.05450766813009977, |
| 0.04650654224678874 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.15836342020581165, |
| "std": 0.005286487688526884, |
| "samples": [ |
| 0.16377839678898454, |
| 0.15809631580486894, |
| 0.1532155480235815 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05368917714804411, |
| "std": 0.00557337638362809, |
| "samples": [ |
| 0.059321336913853884, |
| 0.05356968892738223, |
| 0.048176505602896214 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1689306853028635, |
| "std": 0.005733951335327714, |
| "samples": [ |
| 0.17476224433630705, |
| 0.16873021190986037, |
| 0.16329959966242313 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05444060700635115, |
| "std": 0.006046597427767339, |
| "samples": [ |
| 0.06051499117165804, |
| 0.05438464507460594, |
| 0.04842218477278948 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 24, |
| 24, |
| 24 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1759263346903026, |
| "std": 0.00010780514963377353, |
| "samples": [ |
| 0.17587330704554915, |
| 0.17585531389340758, |
| 0.1760503831319511 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.04769766656681895, |
| "std": 0.00032305769470408497, |
| "samples": [ |
| 0.047570023220032454, |
| 0.04745793156325817, |
| 0.04806504491716623 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 27, |
| 27, |
| 27 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 13598.0, |
| "sampled_peak_conversion_tree_rss_bytes": null, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "IQ3_M", |
| "artifact": { |
| "filename": "occamy-1.0-IQ3_M.gguf", |
| "bytes": 15440519168, |
| "sha256": "dd0ecc62fc13004e3c1c6094559a8d45387eca8d0b843b8c3e971eca1ed479ce" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": true, |
| "new_quantization_tensor_count": 733, |
| "publication_gate": "corrected_copy_verified_see_release_json" |
| }, |
| "measurement_cohort": "initial_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6262.150959, |
| "std_tokens_per_second": 829.11402 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 206.388191, |
| "std_tokens_per_second": 5.201641 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6522.406511, |
| "std_tokens_per_second": 183.466506 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 203.864166, |
| "std_tokens_per_second": 9.484131 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.565706, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.32720599999999944, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 0.332773, |
| "mean_kl": 0.117321, |
| "same_top_percent": 85.931, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 21, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 22, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 0, |
| "base_passed": 22, |
| "plus_passed": 22, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 0 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.28771519179766375, |
| "std": 0.014508686718129687, |
| "samples": [ |
| 0.3010633266530931, |
| 0.2898088186047971, |
| 0.2722734301351011 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06210000952705741, |
| "std": 0.013910054671183586, |
| "samples": [ |
| 0.07498648390173912, |
| 0.06396000692620873, |
| 0.04735353775322437 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 43, |
| 43, |
| 43 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.21264112430314222, |
| "std": 0.005462865029821965, |
| "samples": [ |
| 0.21824755100533366, |
| 0.21234168391674757, |
| 0.20733413798734546 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05440052039921284, |
| "std": 0.007290370283295693, |
| "samples": [ |
| 0.06116812815889716, |
| 0.05535251088440418, |
| 0.04668092215433717 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 30, |
| 30, |
| 30 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.2336125187575817, |
| "std": 0.016227314603196027, |
| "samples": [ |
| 0.2518543777987361, |
| 0.22819991409778595, |
| 0.2207832643762231 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.058666288231809936, |
| "std": 0.013259953752573134, |
| "samples": [ |
| 0.07333162473514676, |
| 0.05514443386346102, |
| 0.04752280609682202 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 33, |
| 33, |
| 33 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.16972966880227128, |
| "std": 0.006189618702843324, |
| "samples": [ |
| 0.17565335799008608, |
| 0.17023103218525648, |
| 0.1633046162314713 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05399713292717934, |
| "std": 0.00624194696431706, |
| "samples": [ |
| 0.059966676868498325, |
| 0.05451026186347008, |
| 0.04751446004956961 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.18053195594499508, |
| "std": 0.00650786843288448, |
| "samples": [ |
| 0.1867452752776444, |
| 0.18108567176386714, |
| 0.17376492079347372 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.054352966137230396, |
| "std": 0.006426498216824092, |
| "samples": [ |
| 0.060608576983213425, |
| 0.05468209274113178, |
| 0.04776822868734598 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 24, |
| 24, |
| 24 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.152838379299889, |
| "std": 0.0010326853413987237, |
| "samples": [ |
| 0.152436182834208, |
| 0.15401164907962084, |
| 0.15206730598583817 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.047985481563955545, |
| "std": 0.0010006312890303353, |
| "samples": [ |
| 0.04764969274401665, |
| 0.04911081911996007, |
| 0.04719593282788992 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 15932.0, |
| "sampled_peak_conversion_tree_rss_bytes": null, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "Q3_K_M", |
| "artifact": { |
| "filename": "occamy-1.0-Q3_K_M.gguf", |
| "bytes": 16764764160, |
| "sha256": "a41137554d3d58e56dc2c043650ff92dbaf12822eb8be4a7a0d05ee1e9717d0c" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": true, |
| "new_quantization_tensor_count": 733, |
| "publication_gate": "corrected_copy_verified_see_release_json" |
| }, |
| "measurement_cohort": "initial_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 5998.353619, |
| "std_tokens_per_second": 807.622891 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 232.862238, |
| "std_tokens_per_second": 2.802633 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 5773.109934, |
| "std_tokens_per_second": 897.221781 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 232.015999, |
| "std_tokens_per_second": 3.481112 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.740173, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.5016730000000003, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 0.50724, |
| "mean_kl": 0.11943, |
| "same_top_percent": 86.115, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 21, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 22, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 0, |
| "base_passed": 22, |
| "plus_passed": 22, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 0 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.2563619092106819, |
| "std": 0.021052974073824043, |
| "samples": [ |
| 0.2616900550201535, |
| 0.2742389137856662, |
| 0.233156758826226 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06968650314956903, |
| "std": 0.020314208797228114, |
| "samples": [ |
| 0.07448101369664073, |
| 0.08717458369210362, |
| 0.04740391205996275 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 39, |
| 39, |
| 39 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.2115073922711114, |
| "std": 0.006295930521400814, |
| "samples": [ |
| 0.21704057324677706, |
| 0.21282450994476676, |
| 0.2046570936217904 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.054443600587546825, |
| "std": 0.006302360700017953, |
| "samples": [ |
| 0.05996254226192832, |
| 0.055792135652154684, |
| 0.04757612384855747 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 33, |
| 33, |
| 33 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.20666205557063222, |
| "std": 0.005896988177183897, |
| "samples": [ |
| 0.2123115500435233, |
| 0.20712924981489778, |
| 0.20054536685347557 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05402073652173082, |
| "std": 0.006035382932927082, |
| "samples": [ |
| 0.05999939516186714, |
| 0.05413262918591499, |
| 0.047930185217410326 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1585510945878923, |
| "std": 0.006215774839613126, |
| "samples": [ |
| 0.16439999220892787, |
| 0.1592292357236147, |
| 0.15202405583113432 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.053471331329395376, |
| "std": 0.0061969726866420455, |
| "samples": [ |
| 0.05939495423808694, |
| 0.053985938895493746, |
| 0.047033100854605436 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.13979922886937857, |
| "std": 0.006068988364186881, |
| "samples": [ |
| 0.14581927098333836, |
| 0.1398959648795426, |
| 0.13368245074525476 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.053526426355044045, |
| "std": 0.0061080691979616035, |
| "samples": [ |
| 0.05962987709790468, |
| 0.053535652812570333, |
| 0.047413749154657125 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 18, |
| 18, |
| 18 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.14381595173229775, |
| "std": 0.0009583601548351085, |
| "samples": [ |
| 0.1427937662228942, |
| 0.14469420490786433, |
| 0.1439598840661347 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.04756571973363558, |
| "std": 0.00036463602550216564, |
| "samples": [ |
| 0.04722997127100825, |
| 0.04795362614095211, |
| 0.04751356178894639 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 17196.0, |
| "sampled_peak_conversion_tree_rss_bytes": null, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "IQ4_XS", |
| "artifact": { |
| "filename": "occamy-1.0-IQ4_XS.gguf", |
| "bytes": 18728777728, |
| "sha256": "c37131d9049f031f209f2190bccb482a1d75ad9f2699ef86b67d1a193dbf1eae" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": true, |
| "new_quantization_tensor_count": 733, |
| "publication_gate": "corrected_copy_verified_see_release_json" |
| }, |
| "measurement_cohort": "initial_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 7037.354404, |
| "std_tokens_per_second": 984.592869 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 227.522137, |
| "std_tokens_per_second": 10.091295 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 7141.300973, |
| "std_tokens_per_second": 228.242693 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 228.189154, |
| "std_tokens_per_second": 6.502732 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.31471, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.07620999999999967, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 0.081777, |
| "mean_kl": 0.047864, |
| "same_top_percent": 91.176, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 19, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 23, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 0, |
| "base_passed": 23, |
| "plus_passed": 23, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 1 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.24785351644580564, |
| "std": 0.022533006700793713, |
| "samples": [ |
| 0.25092874839901924, |
| 0.2686909670010209, |
| 0.22394083393737674 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06964734274273117, |
| "std": 0.0216137923393674, |
| "samples": [ |
| 0.0727028432302177, |
| 0.08957079192623496, |
| 0.046668393071740866 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 37, |
| 37, |
| 37 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.20216554046298066, |
| "std": 0.0060678831557134764, |
| "samples": [ |
| 0.20710565568879247, |
| 0.20399846462532878, |
| 0.19539250107482076 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.0533928998435537, |
| "std": 0.006068543636781278, |
| "samples": [ |
| 0.05854707304388285, |
| 0.05492715071886778, |
| 0.04670447576791048 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 31, |
| 31, |
| 31 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.20751334416369596, |
| "std": 0.00593079165783203, |
| "samples": [ |
| 0.21219580061733723, |
| 0.20949996076524258, |
| 0.2008442711085081 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05386460541437069, |
| "std": 0.005826178248161945, |
| "samples": [ |
| 0.05866871401667595, |
| 0.05554112419486046, |
| 0.04738397803157568 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.15880161896348, |
| "std": 0.00591694626914985, |
| "samples": [ |
| 0.16476176399737597, |
| 0.15871425392106175, |
| 0.15292883897200227 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05297360196709633, |
| "std": 0.005740685834969104, |
| "samples": [ |
| 0.05869632074609399, |
| 0.05300936894491315, |
| 0.04721511621028185 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.14034538576379418, |
| "std": 0.006114715819218857, |
| "samples": [ |
| 0.14566692104563117, |
| 0.14170341100543737, |
| 0.133665825240314 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.053599674720317125, |
| "std": 0.0061441134767715, |
| "samples": [ |
| 0.05892543215304613, |
| 0.05499602761119604, |
| 0.046877564396709204 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 18, |
| 18, |
| 18 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.14290795071671406, |
| "std": 0.0001312851205037561, |
| "samples": [ |
| 0.14293651515617967, |
| 0.14302260195836425, |
| 0.14276473503559828 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.04667325410991907, |
| "std": 0.00010770642693487369, |
| "samples": [ |
| 0.0467379679903388, |
| 0.04673287412151694, |
| 0.04654892021790147 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 19018.0, |
| "sampled_peak_conversion_tree_rss_bytes": null, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "Q4_K_M", |
| "artifact": { |
| "filename": "occamy-1.0-Q4_K_M.gguf", |
| "bytes": 21166757696, |
| "sha256": "ffb25f763ff9c27f5f4e2adcdef399c5654f9f840fdac33dffe7035ba8266a87" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": null, |
| "new_quantization_tensor_count": null, |
| "publication_gate": "unchanged_existing_file_not_part_of_this_publication" |
| }, |
| "measurement_cohort": "initial_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6346.994863, |
| "std_tokens_per_second": 1065.664401 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 241.551554, |
| "std_tokens_per_second": 3.040561 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6311.662919, |
| "std_tokens_per_second": 552.284512 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 238.521534, |
| "std_tokens_per_second": 7.650338 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.242854, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.004354000000000191, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 0.009921, |
| "mean_kl": 0.066201, |
| "same_top_percent": 89.387, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 19, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 24, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 0, |
| "base_passed": 24, |
| "plus_passed": 24, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 1 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.24038906022906303, |
| "std": 0.011171173105804654, |
| "samples": [ |
| 0.24505678098648787, |
| 0.2484693438746035, |
| 0.22764105582609773 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06878846759597461, |
| "std": 0.010579074468882262, |
| "samples": [ |
| 0.07350242510437965, |
| 0.07619113894179463, |
| 0.056671838741749525 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 37, |
| 37, |
| 37 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.16141746751964092, |
| "std": 0.0069305576254149945, |
| "samples": [ |
| 0.16763887973502278, |
| 0.1626660218462348, |
| 0.15394750097766519 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.0544493809963266, |
| "std": 0.00655935901626448, |
| "samples": [ |
| 0.06016011582687497, |
| 0.0559028210118413, |
| 0.04728520615026355 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 23, |
| 23, |
| 23 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.20411067781969905, |
| "std": 0.0038701337013363757, |
| "samples": [ |
| 0.20696542598307133, |
| 0.20566084375604987, |
| 0.19970576371997595 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05501478211954236, |
| "std": 0.0049201494388034055, |
| "samples": [ |
| 0.059163028839975595, |
| 0.05630248971283436, |
| 0.04957882780581713 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.15785645057136813, |
| "std": 0.006728403196999599, |
| "samples": [ |
| 0.1640180773101747, |
| 0.15887407306581736, |
| 0.15067720133811235 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.054961634489397206, |
| "std": 0.006323736960668102, |
| "samples": [ |
| 0.06073369225487113, |
| 0.05594886792823672, |
| 0.04820234328508377 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1396949328482151, |
| "std": 0.006702612691552292, |
| "samples": [ |
| 0.14523081574589014, |
| 0.14161106199026108, |
| 0.1322429208084941 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05537131801247597, |
| "std": 0.006674064226008372, |
| "samples": [ |
| 0.06098819896578789, |
| 0.05713246390223503, |
| 0.047993291169404984 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 18, |
| 18, |
| 18 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.14108201302587986, |
| "std": 0.00027350336194442785, |
| "samples": [ |
| 0.14139112597331405, |
| 0.1408714121207595, |
| 0.14098350098356605 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.04800532308096687, |
| "std": 0.00021959045286995215, |
| "samples": [ |
| 0.04813843499869108, |
| 0.047751869075000286, |
| 0.04812566516920924 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 21916.0, |
| "sampled_peak_conversion_tree_rss_bytes": null, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "Q5_K_M", |
| "artifact": { |
| "filename": "occamy-1.0-Q5_K_M.gguf", |
| "bytes": 24729131008, |
| "sha256": "43cb6af4a64e846a32b3e87502d51008a9a8123bd76f9462120a6dff7ae6c6e4" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": true, |
| "new_quantization_tensor_count": 733, |
| "publication_gate": "corrected_copy_verified_see_release_json" |
| }, |
| "measurement_cohort": "initial_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6388.107699, |
| "std_tokens_per_second": 711.301751 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 237.844422, |
| "std_tokens_per_second": 9.98583 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6310.46296, |
| "std_tokens_per_second": 326.159437 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 238.311167, |
| "std_tokens_per_second": 9.629646 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.248758, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.010257999999999434, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 0.015826, |
| "mean_kl": 0.025792, |
| "same_top_percent": 93.627, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 19, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 23, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 1, |
| "base_passed": 23, |
| "plus_passed": 23, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 1 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.23685449874028563, |
| "std": 0.016028458541619047, |
| "samples": [ |
| 0.24549263576045632, |
| 0.2467110580764711, |
| 0.2183598023839295 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06554047126943867, |
| "std": 0.015546242471568189, |
| "samples": [ |
| 0.07433594577014446, |
| 0.07469505397602916, |
| 0.04759041406214237 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 37, |
| 37, |
| 37 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1836497032394012, |
| "std": 0.006403216372814672, |
| "samples": [ |
| 0.18930969666689634, |
| 0.1849397229962051, |
| 0.1766996900551021 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.054122584871947765, |
| "std": 0.0064213559550823284, |
| "samples": [ |
| 0.05992884561419487, |
| 0.05521311378106475, |
| 0.04722579522058368 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 28, |
| 28, |
| 28 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.20173018674055734, |
| "std": 0.005786127303538893, |
| "samples": [ |
| 0.20760195795446634, |
| 0.20155491633340716, |
| 0.19603368593379855 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05395927963157495, |
| "std": 0.005969194462955443, |
| "samples": [ |
| 0.05992028955370188, |
| 0.05397561518475413, |
| 0.047981934156268835 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1573992989336451, |
| "std": 0.006793295644873734, |
| "samples": [ |
| 0.16212646570056677, |
| 0.16045690421015024, |
| 0.14961452689021826 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.053940510687728725, |
| "std": 0.006182955986684925, |
| "samples": [ |
| 0.060089601669460535, |
| 0.0540076931938529, |
| 0.04772423719987273 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.13717101405685148, |
| "std": 0.00665833142713536, |
| "samples": [ |
| 0.14394385507330298, |
| 0.1369357593357563, |
| 0.1306334277614951 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05355409005035957, |
| "std": 0.006591863368307163, |
| "samples": [ |
| 0.060212945099920034, |
| 0.05341799929738045, |
| 0.04703132575377822 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 18, |
| 18, |
| 18 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.14026402433713278, |
| "std": 0.0006408657264824784, |
| "samples": [ |
| 0.14099251525476575, |
| 0.14001240488141775, |
| 0.13978715287521482 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.04764915940662225, |
| "std": 0.0006587129685173037, |
| "samples": [ |
| 0.04840056225657463, |
| 0.04737567389383912, |
| 0.047171242069453 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 24664.0, |
| "sampled_peak_conversion_tree_rss_bytes": null, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "Q6_K", |
| "artifact": { |
| "filename": "occamy-1.0-Q6_K.gguf", |
| "bytes": 28514152448, |
| "sha256": "43e1b751fe3ebc23bccb72c3183763aabba33741854264033f4e7bb56865c829" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": true, |
| "new_quantization_tensor_count": 733, |
| "publication_gate": "corrected_copy_verified_see_release_json" |
| }, |
| "measurement_cohort": "initial_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 5759.135102, |
| "std_tokens_per_second": 824.856082 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 223.943892, |
| "std_tokens_per_second": 4.211126 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 5765.39998, |
| "std_tokens_per_second": 418.057023 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 221.34997, |
| "std_tokens_per_second": 9.847091 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.259335, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.020834999999999937, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 0.026402, |
| "mean_kl": 0.015831, |
| "same_top_percent": 94.792, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 18, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 24, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 0, |
| "base_passed": 24, |
| "plus_passed": 24, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 1 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.24852733950441083, |
| "std": 0.017025041119127823, |
| "samples": [ |
| 0.2598795732483268, |
| 0.25675075920298696, |
| 0.22895168606191874 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06656573728347819, |
| "std": 0.016371381907610793, |
| "samples": [ |
| 0.07821711525321007, |
| 0.07363214623183012, |
| 0.047847950365394354 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 37, |
| 37, |
| 37 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1979097768974801, |
| "std": 0.005825300530141866, |
| "samples": [ |
| 0.20308292424306273, |
| 0.19904651306569576, |
| 0.19159989338368177 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.055153677705675364, |
| "std": 0.006026722663691082, |
| "samples": [ |
| 0.06085830321535468, |
| 0.05575307784602046, |
| 0.04884965205565095 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 29, |
| 29, |
| 29 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.21267080632969737, |
| "std": 0.006922754916511469, |
| "samples": [ |
| 0.21936996094882488, |
| 0.21309820283204317, |
| 0.20554425520822406 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05530608491972089, |
| "std": 0.0066810864073461144, |
| "samples": [ |
| 0.061676736921072006, |
| 0.055888783652335405, |
| 0.04835273418575525 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.16499523368353644, |
| "std": 0.008838159406532998, |
| "samples": [ |
| 0.1741663608700037, |
| 0.1642866381444037, |
| 0.15653270203620195 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05495617507646481, |
| "std": 0.006580358907119822, |
| "samples": [ |
| 0.061120184138417244, |
| 0.055721879936754704, |
| 0.04802646115422249 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.14352990267798305, |
| "std": 0.006660642715633453, |
| "samples": [ |
| 0.15003244299441576, |
| 0.14383558183908463, |
| 0.13672168320044875 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.054734770519038044, |
| "std": 0.006621428598726721, |
| "samples": [ |
| 0.061189260333776474, |
| 0.05505689000710845, |
| 0.0479581612162292 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 18, |
| 18, |
| 18 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.14678852120414376, |
| "std": 0.00044233899772274494, |
| "samples": [ |
| 0.1464279848150909, |
| 0.14728211658075452, |
| 0.14665546221658587 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.04816233723734816, |
| "std": 0.00034281975884013777, |
| "samples": [ |
| 0.047927014995366335, |
| 0.04855566658079624, |
| 0.0480043301358819 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 28210.0, |
| "sampled_peak_conversion_tree_rss_bytes": null, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "Q8_0", |
| "artifact": { |
| "filename": "occamy-1.0-Q8_0.gguf", |
| "bytes": 36903139456, |
| "sha256": "9bd6eceae8434098c8c7ebec33e4ca0d0bd3f004c2bddf37d16cd906448fdbdd" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": null, |
| "new_quantization_tensor_count": null, |
| "publication_gate": "unchanged_existing_file_not_part_of_this_publication" |
| }, |
| "measurement_cohort": "initial_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6719.326314, |
| "std_tokens_per_second": 1768.730104 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 223.50401, |
| "std_tokens_per_second": 4.416177 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 7072.92687, |
| "std_tokens_per_second": 340.434418 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 223.213252, |
| "std_tokens_per_second": 4.714636 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.257434, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.018933999999999784, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 0.024501, |
| "mean_kl": 0.012633, |
| "same_top_percent": 95.453, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 18, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 23, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 1, |
| "base_passed": 23, |
| "plus_passed": 23, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 2 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.2459441895286242, |
| "std": 0.014663235738846155, |
| "samples": [ |
| 0.25649824598804116, |
| 0.25213317200541496, |
| 0.22920115059241652 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06376485604171951, |
| "std": 0.014135603953686677, |
| "samples": [ |
| 0.07462143991142511, |
| 0.06889195321127772, |
| 0.04778117500245571 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 37, |
| 37, |
| 37 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.18794798214609423, |
| "std": 0.006179040525984396, |
| "samples": [ |
| 0.19346647569909692, |
| 0.1891053938306868, |
| 0.181272076908499 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05491731191674868, |
| "std": 0.006045734620657219, |
| "samples": [ |
| 0.06032932084053755, |
| 0.05603014584630728, |
| 0.04839246906340122 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 27, |
| 27, |
| 27 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.21181433197731772, |
| "std": 0.006002262869676362, |
| "samples": [ |
| 0.2177629149518907, |
| 0.2119202888570726, |
| 0.2057597921229899 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.0546454853999118, |
| "std": 0.005965737947834603, |
| "samples": [ |
| 0.060577703174203634, |
| 0.054711970034986734, |
| 0.048646782990545034 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1627510185353458, |
| "std": 0.005530989530811421, |
| "samples": [ |
| 0.1680593630298972, |
| 0.1631722250021994, |
| 0.15702146757394075 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05425141534457604, |
| "std": 0.005587700042181537, |
| "samples": [ |
| 0.059780027251690626, |
| 0.05436777416616678, |
| 0.048606444615870714 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1728828949853778, |
| "std": 0.00605541141079351, |
| "samples": [ |
| 0.17855435889214277, |
| 0.1735888971015811, |
| 0.1665054289624095 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05464269003520409, |
| "std": 0.00599772028006368, |
| "samples": [ |
| 0.06033726129680872, |
| 0.05520883994176984, |
| 0.04838196886703372 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 24, |
| 24, |
| 24 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.14644432999193668, |
| "std": 0.0005001441619092046, |
| "samples": [ |
| 0.14639546303078532, |
| 0.14597041299566627, |
| 0.14696711394935846 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.047828994070490204, |
| "std": 0.0004616145234633238, |
| "samples": [ |
| 0.047815748024731874, |
| 0.04737414512783289, |
| 0.04829708905890584 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 36432.0, |
| "sampled_peak_conversion_tree_rss_bytes": null, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "IQ3_XS", |
| "artifact": { |
| "filename": "occamy-1.0-IQ3_XS.gguf", |
| "bytes": 14484144128, |
| "sha256": "126b5265fdb52dc53b69be5535c436267cd83acdc87f1820a02ade728d3f5ef4" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": true, |
| "new_quantization_tensor_count": 733, |
| "publication_gate": "corrected_copy_verified_see_release_json" |
| }, |
| "measurement_cohort": "parallel_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6166.288001, |
| "std_tokens_per_second": 1164.815915 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 203.790601, |
| "std_tokens_per_second": 2.581116 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6518.855174, |
| "std_tokens_per_second": 339.58132 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 203.767644, |
| "std_tokens_per_second": 3.585604 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.597194, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.35869399999999985, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 0.364261, |
| "mean_kl": 0.134423, |
| "same_top_percent": 85.025, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 19, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 24, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 1, |
| "base_passed": 24, |
| "plus_passed": 24, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 3 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.3279776278262337, |
| "std": 0.025276325665750526, |
| "samples": [ |
| 0.33012155815958977, |
| 0.35211370326578617, |
| 0.3016976220533252 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.0735549380381902, |
| "std": 0.024266229164964494, |
| "samples": [ |
| 0.07595714088529348, |
| 0.09653072524815798, |
| 0.048176947981119156 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 48, |
| 48, |
| 48 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.21508905695130429, |
| "std": 0.005795080106844155, |
| "samples": [ |
| 0.220191759057343, |
| 0.216286676004529, |
| 0.20878873579204082 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05587405587236086, |
| "std": 0.0060829912063510684, |
| "samples": [ |
| 0.061314751859754324, |
| 0.057001031935214996, |
| 0.049306383822113276 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 30, |
| 30, |
| 30 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.22494482280065617, |
| "std": 0.007589200306552562, |
| "samples": [ |
| 0.2328932760283351, |
| 0.2241663010790944, |
| 0.21777489129453897 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05563883235057195, |
| "std": 0.007136297036218071, |
| "samples": [ |
| 0.06305283913388848, |
| 0.05504648992791772, |
| 0.04881716798990965 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.17111475750183067, |
| "std": 0.006554212135615407, |
| "samples": [ |
| 0.17731431685388088, |
| 0.1717742057517171, |
| 0.164255749899894 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.055112294697513185, |
| "std": 0.006533093013387528, |
| "samples": [ |
| 0.06127970293164253, |
| 0.055790719110518694, |
| 0.04826646205037832 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.15068098731959859, |
| "std": 0.006252782206673077, |
| "samples": [ |
| 0.156892875675112, |
| 0.15076198801398277, |
| 0.144388098269701 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.055363136964539685, |
| "std": 0.006296196070956357, |
| "samples": [ |
| 0.06161465495824814, |
| 0.05545156169682741, |
| 0.04902319423854351 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 18, |
| 18, |
| 18 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.15474400902166963, |
| "std": 0.0005149841880116399, |
| "samples": [ |
| 0.15432095481082797, |
| 0.15531744435429573, |
| 0.15459362789988518 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.04884775805597504, |
| "std": 0.00046250777117193677, |
| "samples": [ |
| 0.04848006879910827, |
| 0.04936703806743026, |
| 0.048696167301386595 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 15020.0, |
| "sampled_peak_conversion_tree_rss_bytes": 3942006784, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "IQ4_NL", |
| "artifact": { |
| "filename": "occamy-1.0-IQ4_NL.gguf", |
| "bytes": 19779278848, |
| "sha256": "6dcf068e182a482b0bdbc493ba7154712a034825d47ebf348d9dee7bece007c0" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": true, |
| "new_quantization_tensor_count": 733, |
| "publication_gate": "corrected_copy_verified_see_release_json" |
| }, |
| "measurement_cohort": "parallel_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6856.118179, |
| "std_tokens_per_second": 1322.337721 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 241.737765, |
| "std_tokens_per_second": 12.595675 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6910.829549, |
| "std_tokens_per_second": 666.879687 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 244.747027, |
| "std_tokens_per_second": 3.578141 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.318163, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.07966300000000004, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 0.08523, |
| "mean_kl": 0.046354, |
| "same_top_percent": 91.238, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 19, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 23, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 1, |
| "base_passed": 23, |
| "plus_passed": 23, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 3 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.23681716822708646, |
| "std": 0.019683988387632887, |
| "samples": [ |
| 0.24371080193668604, |
| 0.2521271570585668, |
| 0.21461354568600655 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06833163400491078, |
| "std": 0.019009738096268068, |
| "samples": [ |
| 0.07510404428467155, |
| 0.08302776794880629, |
| 0.04686308978125453 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 37, |
| 37, |
| 37 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.17535486879448095, |
| "std": 0.005153032552155704, |
| "samples": [ |
| 0.17990116402506828, |
| 0.17640619771555066, |
| 0.16975724464282393 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05686028953641653, |
| "std": 0.005137851382395989, |
| "samples": [ |
| 0.061482592951506376, |
| 0.05776989087462425, |
| 0.05132838478311896 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 26, |
| 26, |
| 26 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.20158252958208323, |
| "std": 0.007303588719258058, |
| "samples": [ |
| 0.2088620811700821, |
| 0.2016303688287735, |
| 0.19425513874739408 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.056268551697333656, |
| "std": 0.006735278909863007, |
| "samples": [ |
| 0.06282382505014539, |
| 0.05661517707630992, |
| 0.049366652965545654 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.15565358443806568, |
| "std": 0.005998443204966175, |
| "samples": [ |
| 0.1613878458738327, |
| 0.15615097526460886, |
| 0.1494219321757555 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.055379202434172235, |
| "std": 0.006029209991806074, |
| "samples": [ |
| 0.06116515304893255, |
| 0.0558393532410264, |
| 0.049133101012557745 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.13901671037698785, |
| "std": 0.006444154475504606, |
| "samples": [ |
| 0.14267711620777845, |
| 0.14279705099761486, |
| 0.13157596392557025 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05499925510957837, |
| "std": 0.0057071932300593025, |
| "samples": [ |
| 0.06045859307050705, |
| 0.055466268211603165, |
| 0.0490729040466249 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 18, |
| 18, |
| 18 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.14053894253447652, |
| "std": 0.0003433460067537544, |
| "samples": [ |
| 0.1409282316453755, |
| 0.14040931314229965, |
| 0.1402792828157544 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.04910820738101999, |
| "std": 0.00029116060117625943, |
| "samples": [ |
| 0.04940951196476817, |
| 0.04908672999590635, |
| 0.048828380182385445 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 20, |
| 20, |
| 20 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 20004.0, |
| "sampled_peak_conversion_tree_rss_bytes": 3941294080, |
| "file_size_is_not_runtime_memory": true |
| } |
| }, |
| { |
| "type": "Q4_K_S", |
| "artifact": { |
| "filename": "occamy-1.0-Q4_K_S.gguf", |
| "bytes": 19889903616, |
| "sha256": "d7b4dc1b3f8b3be6fa0a16eda8bfb613119afa5a7c8db49d823b59b8af988b0d" |
| }, |
| "validation": { |
| "load_tested": true, |
| "benchmarked": true, |
| "new_quantization_full_finite_scan_passed": true, |
| "new_quantization_tensor_count": 733, |
| "publication_gate": "corrected_copy_verified_see_release_json" |
| }, |
| "measurement_cohort": "parallel_single_gpu_jobs", |
| "engine": { |
| "pp512": { |
| "kind": "pp", |
| "prompt_tokens": 512, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6305.7643, |
| "std_tokens_per_second": 1593.437097 |
| }, |
| "tg128_d512": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 512, |
| "mean_tokens_per_second": 239.666664, |
| "std_tokens_per_second": 4.069765 |
| }, |
| "pp2048": { |
| "kind": "pp", |
| "prompt_tokens": 2048, |
| "generated_tokens": 0, |
| "decode_initial_context_tokens": null, |
| "mean_tokens_per_second": 6584.668241, |
| "std_tokens_per_second": 270.456255 |
| }, |
| "tg128_d2048": { |
| "kind": "tg", |
| "prompt_tokens": 0, |
| "generated_tokens": 128, |
| "decode_initial_context_tokens": 2048, |
| "mean_tokens_per_second": 237.581923, |
| "std_tokens_per_second": 6.619769 |
| } |
| }, |
| "ppl": { |
| "ppl": 6.282276, |
| "direct_bf16_reference_ppl": 6.2385, |
| "ppl_delta_vs_direct_bf16": 0.04377600000000026, |
| "dump_base_ppl": 6.232933, |
| "paired_ppl_delta": 0.049343, |
| "mean_kl": 0.051069, |
| "same_top_percent": 90.6, |
| "chunks": 32, |
| "n_ctx": 512, |
| "scored_tokens": 8160 |
| }, |
| "quality": { |
| "math_strict": { |
| "passed": 0, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 24 |
| }, |
| "math_flexible": { |
| "passed": 21, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": null, |
| "truncated": null, |
| "base_passed": null, |
| "plus_passed": null, |
| "extraction_invalid": 0 |
| }, |
| "code": { |
| "passed": 23, |
| "scored": 24, |
| "total_planned": 24, |
| "pending_or_missing": 0, |
| "truncated": 0, |
| "base_passed": 23, |
| "plus_passed": 23, |
| "extraction_invalid": null |
| }, |
| "code_outputs_actually_executed": 24, |
| "json": { |
| "passed": 4, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "tool": { |
| "passed": 6, |
| "scored": 6, |
| "planned": 6, |
| "pending": 0, |
| "truncated": 0 |
| }, |
| "request_errors": 0, |
| "truncations": 0 |
| }, |
| "e2e": { |
| "text_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.4898868848880132, |
| "std": 0.016192211295917417, |
| "samples": [ |
| 0.5000728080049157, |
| 0.49837232287973166, |
| 0.47121552377939224 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.06592472270131111, |
| "std": 0.01537102868090097, |
| "samples": [ |
| 0.07626282703131437, |
| 0.07325011817738414, |
| 0.04826122289523482 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 92, |
| 92, |
| 92 |
| ] |
| }, |
| "text_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.17950111270571747, |
| "std": 0.006368340839696904, |
| "samples": [ |
| 0.1853360990062356, |
| 0.18045924417674541, |
| 0.17270799493417144 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.054418385649720825, |
| "std": 0.0062758226975835945, |
| "samples": [ |
| 0.06013060500845313, |
| 0.05542412307113409, |
| 0.04770042886957526 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 27, |
| 27, |
| 27 |
| ] |
| }, |
| "code_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.20182643691077828, |
| "std": 0.006497579709304292, |
| "samples": [ |
| 0.20816185884177685, |
| 0.20213943906128407, |
| 0.19517801282927394 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.053930392023175955, |
| "std": 0.006146241344660085, |
| "samples": [ |
| 0.059900789987295866, |
| 0.05426815012469888, |
| 0.04762223595753312 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 32, |
| 32, |
| 32 |
| ] |
| }, |
| "code_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1552351638674736, |
| "std": 0.005967008568245169, |
| "samples": [ |
| 0.1610678369179368, |
| 0.15549532463774085, |
| 0.14914233004674315 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.05341656723370155, |
| "std": 0.0060258322134452424, |
| "samples": [ |
| 0.059284294955432415, |
| 0.05372121883556247, |
| 0.04724418791010976 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 22, |
| 22, |
| 22 |
| ] |
| }, |
| "json_0": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.16569887846708298, |
| "std": 0.005754651722380783, |
| "samples": [ |
| 0.17087450018152595, |
| 0.16672018403187394, |
| 0.15950195118784904 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.0542328303369383, |
| "std": 0.005841838087165158, |
| "samples": [ |
| 0.05981520004570484, |
| 0.05472111701965332, |
| 0.04816217394545674 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 24, |
| 24, |
| 24 |
| ] |
| }, |
| "json_1": { |
| "requests": 3, |
| "errors": 0, |
| "truncated": 0, |
| "latency_s": { |
| "n": 3, |
| "mean": 0.1732787311387559, |
| "std": 0.00031225285517904, |
| "samples": [ |
| 0.17363912938162684, |
| 0.17310784198343754, |
| 0.17308922205120325 |
| ] |
| }, |
| "ttft_s": { |
| "n": 3, |
| "mean": 0.04807448045661052, |
| "std": 0.00024927805570199686, |
| "samples": [ |
| 0.04830955620855093, |
| 0.04781308816745877, |
| 0.04810079699382186 |
| ] |
| }, |
| "actual_output_tokens": [ |
| 27, |
| 27, |
| 27 |
| ] |
| } |
| }, |
| "memory": { |
| "sampled_peak_device_used_mib": 20110.0, |
| "sampled_peak_conversion_tree_rss_bytes": 3940864000, |
| "file_size_is_not_runtime_memory": true |
| } |
| } |
| ], |
| "release_artifacts": "release.json", |
| "benchmark_artifact_hashes_are_pre_correction": true |
| } |
|
|
