diff --git a/.gitignore b/.gitignore index d47420a..d474028 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ niah_results*/ scripts/.turbo-diag-venv/ niah_decay_test/ niah_v2_results/ +wikitext-103-raw/ +benchmark-results-raw/*.kld diff --git a/README.md b/README.md index 6eecfad..d2a7383 100644 --- a/README.md +++ b/README.md @@ -37,37 +37,64 @@ Validated end-to-end on Qwen 3.5 35B-A3B (MoE) on M5 Max via llama.cpp Metal. ### Top-of-Tree Results -| Cache Type | Compression | Prefill tok/s | PPL (wikitext-2) | vs q8_0 prefill | -|------------|-------------|--------------|-------------------|---------------| -| f16 | 1.0x | — | 6.121 | — | -| q8_0 | 2.0x | 2694 | 5.414 | baseline | -| q4_0 | 4.0x | — | 6.142 | — | -| **turbo3** | **4.6x** | **2747** | **5.445** | **1.02x** | +| Cache Type | Bits/val | Compression | PPL (wikitext-2, 512c) | vs q8_0 | +|------------|----------|-------------|----------------------|---------| +| f16 | 16.0 | 1.0x | 6.121 | -0.16% | +| q8_0 | 8.5 | 1.9x | 6.111 | baseline | +| **turbo4** | **4.25** | **3.8x** | **6.125** | **+0.23%** | +| q4_0 | 4.5 | 3.6x | 6.142 | +0.52% | +| turbo3 | 3.5 | 4.6x | 6.176 | +1.06% | -4.6x compression. ~1% perplexity increase vs q8_0 due to compression; sparse V introduces no additional degradation. +turbo4 (4-bit PolarQuant) has the best quality after q8_0 — closer to q8_0 than q4_0, at better compression. turbo3 trades quality for maximum compression. -### Context Scaling (Verified 2K-32K) +### Prefill Context Scaling (Verified 2K-32K) -| Context | turbo3 tok/s | q8_0 tok/s | turbo3/q8_0 | -|---------|-------------|-----------|-------------| -| 2K | 4694 | 4756 | 0.987x | -| 4K | 3049 | 3084 | 0.989x | -| 8K | 2287 | 2299 | 0.995x | -| 16K | 1737 | 1757 | 0.989x | -| 32K | 1211 | 1217 | 0.995x | +| Context | turbo4 tok/s | turbo3 tok/s | q8_0 tok/s | turbo4/q8_0 | turbo3/q8_0 | +|---------|-------------|-------------|-----------|------------|------------| +| 2K | 2682 | 2708 | 2665 | 1.01x | 1.02x | +| 4K | 2370 | 2289 | 2255 | 1.05x | 1.01x | +| 8K | 2041 | 2054 | 2002 | 1.02x | 1.03x | +| 16K | 1621 | 1698 | 1605 | 1.01x | 1.06x | +| 32K | 1141 | 1204 | 1098 | 1.04x | 1.10x | -**Prefill: flat 99% of q8_0 speed regardless of context length.** +**Prefill: both turbo3 and turbo4 match or exceed q8_0 speed.** Compressed cache uses less bandwidth. ### Decode Speed — MoE (M5 Max 128GB, Qwen3.5-35B-A3B, Sparse V) -| Context | turbo3 decode | q8_0 decode | turbo3/q8_0 | -|---------|-------------|-----------|-------------| -| Short (~12 tok) | 77.6 | 86.3 | 0.90x | -| 4K | 74.9 | — | — | -| 8K | 71.7 | — | — | -| 16K | 66.5 | 72.0 | 0.92x | -| 24K (70-page PDF) | 53.3 | 68.2 | 0.78x | -| 32K | 57.7 | 62.0 | **0.93x** | +| Config | Short decode (tg128) | pp32768+tg128 | Short vs q8_0 | +|--------|---------------------|---------------|--------------| +| q8_0 | 85.71 tok/s | 1173.91 tok/s | — | +| **turbo4** | **79.87 tok/s** | **1060.12 tok/s** | **0.93x** | +| turbo3 | 76.84 tok/s | 1141.74 tok/s | 0.90x | + +turbo4 decode is faster than turbo3 due to simpler nibble packing and direct-extract dequant. + +**Real-world server benchmark (70-page PDF, ~24K context):** + +| Config | Prefill tok/s | Decode tok/s | Decode vs q8_0 | +|--------|-------------|-------------|---------------| +| q8_0 | 1449.9 | 68.2 | — | +| turbo4 | 1405.9 | 63.7 | 0.93x | +| turbo3 | 1417.8 | 53.3 | 0.78x | + +### NIAH Retrieval (turbo4) + +| Test | q8_0 | turbo4 | turbo3 + sparse V | +|------|------|--------|-------------------| +| Single needle (9 positions) | 7/9 | 7/9 | 9/9 | + +turbo4 matches q8_0 retrieval accuracy. + +### KL Divergence vs f16 + +| Cache | Mean KLD | Δp RMS | Same top-p % | +|-------|----------|--------|-------------| +| q8_0 | 0.001549 | 1.23% | 98.43% | +| **turbo4** | **0.009633** | **2.71%** | **95.98%** | +| q4_0 | 0.008091 | 2.75% | 95.83% | +| turbo3 | 0.016145 | 4.09% | 94.31% | + +turbo4 KLD is 40% lower than turbo3. Same top-p agreement matches q4_0. ### Decode Speed — Dense (M5 Max 128GB, Qwen3.5-27B, Sparse V) diff --git a/benchmark-results-raw/kld_dense_q4_0.log b/benchmark-results-raw/kld_dense_q4_0.log new file mode 100644 index 0000000..b94aa64 --- /dev/null +++ b/benchmark-results-raw/kld_dense_q4_0.log @@ -0,0 +1,270 @@ +ggml_metal_device_init: testing tensor API for f16 support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x104561880 | th_max = 1024 | th_width = 32 +ggml_metal_device_init: testing tensor API for bfloat support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x104561500 | th_max = 1024 | th_width = 32 +ggml_metal_library_init: using embedded metal library +ggml_metal_library_init: turbo3 sparse V dequant enabled +ggml_metal_library_init: loaded in 0.008 sec +ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s) +ggml_metal_device_init: GPU name: MTL0 +ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010) +ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003) +ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002) +ggml_metal_device_init: simdgroup reduction = true +ggml_metal_device_init: simdgroup matrix mul. = true +ggml_metal_device_init: has unified memory = true +ggml_metal_device_init: has bfloat = true +ggml_metal_device_init: has tensor = true +ggml_metal_device_init: use residency sets = true +ggml_metal_device_init: use shared buffers = true +ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB +build: 159 (7d1bd95) with AppleClang 17.0.0.17000604 for Darwin arm64 +common_init_result: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on +llama_params_fit_impl: projected to use 27101 MiB of device memory vs. 109501 MiB of free device memory +llama_params_fit_impl: will leave 82399 >= 1024 MiB of free device memory, no changes needed +llama_params_fit: successfully fit params to free device memory +llama_params_fit: fitting params to free memory took 0.44 seconds +llama_model_load_from_file_impl: using device MTL0 (Apple M5 Max) (unknown id) - 110100 MiB free +llama_model_loader: loaded meta data with 49 key-value pairs and 851 tensors from /Users/tom/local_llms/models/Qwen3.5-27B-Q8_0.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35 +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +llama_model_loader: - kv 4: general.sampling.temp f32 = 0.600000 +llama_model_loader: - kv 5: general.name str = Qwen3.5-27B +llama_model_loader: - kv 6: general.basename str = Qwen3.5-27B +llama_model_loader: - kv 7: general.quantized_by str = Unsloth +llama_model_loader: - kv 8: general.size_label str = 27B +llama_model_loader: - kv 9: general.license str = apache-2.0 +llama_model_loader: - kv 10: general.license.link str = https://huggingface.co/Qwen/Qwen3.5-2... +llama_model_loader: - kv 11: general.repo_url str = https://huggingface.co/unsloth +llama_model_loader: - kv 12: general.base_model.count u32 = 1 +llama_model_loader: - kv 13: general.base_model.0.name str = Qwen3.5 27B +llama_model_loader: - kv 14: general.base_model.0.organization str = Qwen +llama_model_loader: - kv 15: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen3.5-27B +llama_model_loader: - kv 16: general.tags arr[str,3] = ["qwen3_5_moe", "unsloth", "image-tex... +llama_model_loader: - kv 17: qwen35.block_count u32 = 64 +llama_model_loader: - kv 18: qwen35.context_length u32 = 262144 +llama_model_loader: - kv 19: qwen35.embedding_length u32 = 5120 +llama_model_loader: - kv 20: qwen35.feed_forward_length u32 = 17408 +llama_model_loader: - kv 21: qwen35.attention.head_count u32 = 24 +llama_model_loader: - kv 22: qwen35.attention.head_count_kv u32 = 4 +llama_model_loader: - kv 23: qwen35.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 24: qwen35.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 25: qwen35.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 26: qwen35.attention.key_length u32 = 256 +llama_model_loader: - kv 27: qwen35.attention.value_length u32 = 256 +llama_model_loader: - kv 28: qwen35.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 29: qwen35.ssm.state_size u32 = 128 +llama_model_loader: - kv 30: qwen35.ssm.group_count u32 = 16 +llama_model_loader: - kv 31: qwen35.ssm.time_step_rank u32 = 48 +llama_model_loader: - kv 32: qwen35.ssm.inner_size u32 = 6144 +llama_model_loader: - kv 33: qwen35.full_attention_interval u32 = 4 +llama_model_loader: - kv 34: qwen35.rope.dimension_count u32 = 64 +llama_model_loader: - kv 35: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 36: tokenizer.ggml.pre str = qwen35 +llama_model_loader: - kv 37: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 38: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 39: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 40: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 41: tokenizer.ggml.padding_token_id u32 = 248055 +llama_model_loader: - kv 42: tokenizer.chat_template str = {%- set image_count = namespace(value... +llama_model_loader: - kv 43: general.quantization_version u32 = 2 +llama_model_loader: - kv 44: general.file_type u32 = 7 +llama_model_loader: - kv 45: quantize.imatrix.file str = Qwen3.5-27B-GGUF/imatrix_unsloth.gguf +llama_model_loader: - kv 46: quantize.imatrix.dataset str = unsloth_calibration_Qwen3.5-27B.txt +llama_model_loader: - kv 47: quantize.imatrix.entries_count u32 = 496 +llama_model_loader: - kv 48: quantize.imatrix.chunks_count u32 = 80 +llama_model_loader: - type f32: 353 tensors +llama_model_loader: - type q8_0: 498 tensors +print_info: file format = GGUF V3 (latest) +print_info: file type = Q8_0 +print_info: file size = 26.62 GiB (8.50 BPW) +load: 0 unused tokens +load: printing all EOG tokens: +load: - 248044 ('<|endoftext|>') +load: - 248046 ('<|im_end|>') +load: - 248063 ('<|fim_pad|>') +load: - 248064 ('<|repo_name|>') +load: - 248065 ('<|file_sep|>') +load: special tokens cache size = 33 +load: token to piece cache size = 1.7581 MB +print_info: arch = qwen35 +print_info: vocab_only = 0 +print_info: no_alloc = 0 +print_info: n_ctx_train = 262144 +print_info: n_embd = 5120 +print_info: n_embd_inp = 5120 +print_info: n_layer = 64 +print_info: n_head = 24 +print_info: n_head_kv = 4 +print_info: n_rot = 64 +print_info: n_swa = 0 +print_info: is_swa_any = 0 +print_info: n_embd_head_k = 256 +print_info: n_embd_head_v = 256 +print_info: n_gqa = 6 +print_info: n_embd_k_gqa = 1024 +print_info: n_embd_v_gqa = 1024 +print_info: f_norm_eps = 0.0e+00 +print_info: f_norm_rms_eps = 1.0e-06 +print_info: f_clamp_kqv = 0.0e+00 +print_info: f_max_alibi_bias = 0.0e+00 +print_info: f_logit_scale = 0.0e+00 +print_info: f_attn_scale = 0.0e+00 +print_info: n_ff = 17408 +print_info: n_expert = 0 +print_info: n_expert_used = 0 +print_info: n_expert_groups = 0 +print_info: n_group_used = 0 +print_info: causal attn = 1 +print_info: pooling type = -1 +print_info: rope type = 40 +print_info: rope scaling = linear +print_info: freq_base_train = 10000000.0 +print_info: freq_scale_train = 1 +print_info: n_ctx_orig_yarn = 262144 +print_info: rope_yarn_log_mul = 0.0000 +print_info: rope_finetuned = unknown +print_info: mrope sections = [11, 11, 10, 0] +print_info: ssm_d_conv = 4 +print_info: ssm_d_inner = 6144 +print_info: ssm_d_state = 128 +print_info: ssm_dt_rank = 48 +print_info: ssm_n_group = 16 +print_info: ssm_dt_b_c_rms = 0 +print_info: model type = 27B +print_info: model params = 26.90 B +print_info: general.name = Qwen3.5-27B +print_info: vocab type = BPE +print_info: n_vocab = 248320 +print_info: n_merges = 247587 +print_info: BOS token = 11 ',' +print_info: EOS token = 248046 '<|im_end|>' +print_info: EOT token = 248046 '<|im_end|>' +print_info: PAD token = 248055 '<|vision_pad|>' +print_info: LF token = 198 'Ċ' +print_info: FIM PRE token = 248060 '<|fim_prefix|>' +print_info: FIM SUF token = 248062 '<|fim_suffix|>' +print_info: FIM MID token = 248061 '<|fim_middle|>' +print_info: FIM PAD token = 248063 '<|fim_pad|>' +print_info: FIM REP token = 248064 '<|repo_name|>' +print_info: FIM SEP token = 248065 '<|file_sep|>' +print_info: EOG token = 248044 '<|endoftext|>' +print_info: EOG token = 248046 '<|im_end|>' +print_info: EOG token = 248063 '<|fim_pad|>' +print_info: EOG token = 248064 '<|repo_name|>' +print_info: EOG token = 248065 '<|file_sep|>' +print_info: max token length = 256 +load_tensors: loading model tensors, this can take a while... (mmap = true, direct_io = false) +load_tensors: offloading output layer to GPU +load_tensors: offloading 63 repeating layers to GPU +load_tensors: offloaded 65/65 layers to GPU +load_tensors: CPU_Mapped model buffer size = 1288.28 MiB +load_tensors: MTL0_Mapped model buffer size = 27260.56 MiB +............................................................................................. +common_init_result: added <|endoftext|> logit bias = -inf +common_init_result: added <|im_end|> logit bias = -inf +common_init_result: added <|fim_pad|> logit bias = -inf +common_init_result: added <|repo_name|> logit bias = -inf +common_init_result: added <|file_sep|> logit bias = -inf +llama_context: constructing llama_context +llama_context: n_seq_max = 4 +llama_context: n_ctx = 2048 +llama_context: n_ctx_seq = 512 +llama_context: n_batch = 2048 +llama_context: n_ubatch = 512 +llama_context: causal_attn = 1 +llama_context: flash_attn = enabled +llama_context: kv_unified = false +llama_context: freq_base = 10000000.0 +llama_context: freq_scale = 1 +llama_context: n_ctx_seq (512) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +ggml_metal_init: allocating +ggml_metal_init: found device: Apple M5 Max +ggml_metal_init: picking default device: Apple M5 Max +ggml_metal_init: use fusion = true +ggml_metal_init: use concurrency = true +ggml_metal_init: use graph optimize = true +llama_context: CPU output buffer size = 3.79 MiB +llama_kv_cache: MTL0 KV buffer size = 36.00 MiB +llama_kv_cache: size = 36.00 MiB ( 512 cells, 16 layers, 4/4 seqs), K (q4_0): 18.00 MiB, V (q4_0): 18.00 MiB +llama_memory_recurrent: MTL0 RS buffer size = 598.50 MiB +llama_memory_recurrent: size = 598.50 MiB ( 4 cells, 64 layers, 4 seqs), R (f32): 22.50 MiB, S (f32): 576.00 MiB +sched_reserve: reserving ... +sched_reserve: resolving fused Gated Delta Net support: +sched_reserve: fused Gated Delta Net (autoregressive) enabled +sched_reserve: fused Gated Delta Net (chunked) enabled +sched_reserve: MTL0 compute buffer size = 495.00 MiB +sched_reserve: CPU compute buffer size = 21.02 MiB +sched_reserve: graph nodes = 3689 +sched_reserve: graph splits = 2 +sched_reserve: reserve took 21.91 ms, sched copies = 1 +common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) + +system_info: n_threads = 6 (n_threads_batch = 6) / 18 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | REPACK = 1 | +kl_divergence: computing over 8 chunks, n_ctx=512, batch_size=2048, n_seq=4 +kl_divergence: 4.24 seconds per pass - ETA 0.13 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 4.3701 ± 0.6250 -0.00411 ± 0.00599 0.00272 ± 0.00033 1.726 ± 0.239 % 96.863 ± 1.094 % + 2 6.1576 ± 0.7183 -0.00785 ± 0.00383 0.00311 ± 0.00027 1.566 ± 0.144 % 97.255 ± 0.724 % + 3 6.1263 ± 0.5853 -0.00665 ± 0.00545 0.00303 ± 0.00021 1.487 ± 0.111 % 97.124 ± 0.605 % + 4 6.1484 ± 0.4897 -0.00398 ± 0.00431 0.00319 ± 0.00027 1.646 ± 0.224 % 97.353 ± 0.503 % + 5 6.0643 ± 0.4175 -0.00294 ± 0.00357 0.00298 ± 0.00022 1.580 ± 0.188 % 97.490 ± 0.438 % + 6 6.2810 ± 0.3940 -0.00177 ± 0.00304 0.00287 ± 0.00018 1.527 ± 0.164 % 97.778 ± 0.377 % + 7 6.4068 ± 0.3694 -0.00028 ± 0.00281 0.00281 ± 0.00016 1.485 ± 0.145 % 97.647 ± 0.359 % + 8 6.8788 ± 0.3816 0.00086 ± 0.00254 0.00274 ± 0.00014 1.437 ± 0.131 % 97.647 ± 0.336 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 6.878806 ± 0.381592 +Mean PPL(base) : 6.872893 ± 0.379370 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.90% +Mean ln(PPL(Q)/PPL(base)) : 0.000860 ± 0.002542 +Mean PPL(Q)/PPL(base) : 1.000860 ± 0.002544 +Mean PPL(Q)-PPL(base) : 0.005913 ± 0.017515 + +====== KL divergence statistics ====== +Mean KLD: 0.002741 ± 0.000142 +Maximum KLD: 0.210398 +99.9% KLD: 0.073911 +99.0% KLD: 0.020144 +95.0% KLD: 0.008533 +90.0% KLD: 0.005757 +Median KLD: 0.001445 +10.0% KLD: 0.000109 + 5.0% KLD: 0.000029 + 1.0% KLD: 0.000003 + 0.1% KLD: -0.000001 +Minimum KLD: -0.000017 + +====== Token probability statistics ====== +Mean Δp: 0.004 ± 0.032 % +Maximum Δp: 26.606% +99.9% Δp: 8.512% +99.0% Δp: 4.052% +95.0% Δp: 1.952% +90.0% Δp: 1.174% +75.0% Δp: 0.292% +Median Δp: -0.000% +25.0% Δp: -0.355% +10.0% Δp: -1.232% + 5.0% Δp: -1.915% + 1.0% Δp: -3.607% + 0.1% Δp: -8.014% +Minimum Δp: -11.468% +RMS Δp : 1.437 ± 0.131 % +Same top p: 97.647 ± 0.336 % + +llama_perf_context_print: load time = 1930.89 ms +llama_perf_context_print: prompt eval time = 4412.57 ms / 4096 tokens ( 1.08 ms per token, 928.26 tokens per second) +llama_perf_context_print: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) +llama_perf_context_print: total time = 8975.85 ms / 4097 tokens +llama_perf_context_print: graphs reused = 2 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - MTL0 (Apple M5 Max) | 110100 = 81709 + (28390 = 27260 + 634 + 495) + 0 | +llama_memory_breakdown_print: | - Host | 1309 = 1288 + 0 + 21 | +ggml_metal_free: deallocating diff --git a/benchmark-results-raw/kld_dense_q8_0.log b/benchmark-results-raw/kld_dense_q8_0.log new file mode 100644 index 0000000..0f441fa --- /dev/null +++ b/benchmark-results-raw/kld_dense_q8_0.log @@ -0,0 +1,270 @@ +ggml_metal_device_init: testing tensor API for f16 support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10261aea0 | th_max = 1024 | th_width = 32 +ggml_metal_device_init: testing tensor API for bfloat support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10261ab20 | th_max = 1024 | th_width = 32 +ggml_metal_library_init: using embedded metal library +ggml_metal_library_init: turbo3 sparse V dequant enabled +ggml_metal_library_init: loaded in 0.009 sec +ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s) +ggml_metal_device_init: GPU name: MTL0 +ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010) +ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003) +ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002) +ggml_metal_device_init: simdgroup reduction = true +ggml_metal_device_init: simdgroup matrix mul. = true +ggml_metal_device_init: has unified memory = true +ggml_metal_device_init: has bfloat = true +ggml_metal_device_init: has tensor = true +ggml_metal_device_init: use residency sets = true +ggml_metal_device_init: use shared buffers = true +ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB +build: 159 (7d1bd95) with AppleClang 17.0.0.17000604 for Darwin arm64 +common_init_result: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on +llama_params_fit_impl: projected to use 27133 MiB of device memory vs. 109501 MiB of free device memory +llama_params_fit_impl: will leave 82367 >= 1024 MiB of free device memory, no changes needed +llama_params_fit: successfully fit params to free device memory +llama_params_fit: fitting params to free memory took 0.30 seconds +llama_model_load_from_file_impl: using device MTL0 (Apple M5 Max) (unknown id) - 110100 MiB free +llama_model_loader: loaded meta data with 49 key-value pairs and 851 tensors from /Users/tom/local_llms/models/Qwen3.5-27B-Q8_0.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35 +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +llama_model_loader: - kv 4: general.sampling.temp f32 = 0.600000 +llama_model_loader: - kv 5: general.name str = Qwen3.5-27B +llama_model_loader: - kv 6: general.basename str = Qwen3.5-27B +llama_model_loader: - kv 7: general.quantized_by str = Unsloth +llama_model_loader: - kv 8: general.size_label str = 27B +llama_model_loader: - kv 9: general.license str = apache-2.0 +llama_model_loader: - kv 10: general.license.link str = https://huggingface.co/Qwen/Qwen3.5-2... +llama_model_loader: - kv 11: general.repo_url str = https://huggingface.co/unsloth +llama_model_loader: - kv 12: general.base_model.count u32 = 1 +llama_model_loader: - kv 13: general.base_model.0.name str = Qwen3.5 27B +llama_model_loader: - kv 14: general.base_model.0.organization str = Qwen +llama_model_loader: - kv 15: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen3.5-27B +llama_model_loader: - kv 16: general.tags arr[str,3] = ["qwen3_5_moe", "unsloth", "image-tex... +llama_model_loader: - kv 17: qwen35.block_count u32 = 64 +llama_model_loader: - kv 18: qwen35.context_length u32 = 262144 +llama_model_loader: - kv 19: qwen35.embedding_length u32 = 5120 +llama_model_loader: - kv 20: qwen35.feed_forward_length u32 = 17408 +llama_model_loader: - kv 21: qwen35.attention.head_count u32 = 24 +llama_model_loader: - kv 22: qwen35.attention.head_count_kv u32 = 4 +llama_model_loader: - kv 23: qwen35.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 24: qwen35.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 25: qwen35.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 26: qwen35.attention.key_length u32 = 256 +llama_model_loader: - kv 27: qwen35.attention.value_length u32 = 256 +llama_model_loader: - kv 28: qwen35.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 29: qwen35.ssm.state_size u32 = 128 +llama_model_loader: - kv 30: qwen35.ssm.group_count u32 = 16 +llama_model_loader: - kv 31: qwen35.ssm.time_step_rank u32 = 48 +llama_model_loader: - kv 32: qwen35.ssm.inner_size u32 = 6144 +llama_model_loader: - kv 33: qwen35.full_attention_interval u32 = 4 +llama_model_loader: - kv 34: qwen35.rope.dimension_count u32 = 64 +llama_model_loader: - kv 35: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 36: tokenizer.ggml.pre str = qwen35 +llama_model_loader: - kv 37: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 38: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 39: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 40: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 41: tokenizer.ggml.padding_token_id u32 = 248055 +llama_model_loader: - kv 42: tokenizer.chat_template str = {%- set image_count = namespace(value... +llama_model_loader: - kv 43: general.quantization_version u32 = 2 +llama_model_loader: - kv 44: general.file_type u32 = 7 +llama_model_loader: - kv 45: quantize.imatrix.file str = Qwen3.5-27B-GGUF/imatrix_unsloth.gguf +llama_model_loader: - kv 46: quantize.imatrix.dataset str = unsloth_calibration_Qwen3.5-27B.txt +llama_model_loader: - kv 47: quantize.imatrix.entries_count u32 = 496 +llama_model_loader: - kv 48: quantize.imatrix.chunks_count u32 = 80 +llama_model_loader: - type f32: 353 tensors +llama_model_loader: - type q8_0: 498 tensors +print_info: file format = GGUF V3 (latest) +print_info: file type = Q8_0 +print_info: file size = 26.62 GiB (8.50 BPW) +load: 0 unused tokens +load: printing all EOG tokens: +load: - 248044 ('<|endoftext|>') +load: - 248046 ('<|im_end|>') +load: - 248063 ('<|fim_pad|>') +load: - 248064 ('<|repo_name|>') +load: - 248065 ('<|file_sep|>') +load: special tokens cache size = 33 +load: token to piece cache size = 1.7581 MB +print_info: arch = qwen35 +print_info: vocab_only = 0 +print_info: no_alloc = 0 +print_info: n_ctx_train = 262144 +print_info: n_embd = 5120 +print_info: n_embd_inp = 5120 +print_info: n_layer = 64 +print_info: n_head = 24 +print_info: n_head_kv = 4 +print_info: n_rot = 64 +print_info: n_swa = 0 +print_info: is_swa_any = 0 +print_info: n_embd_head_k = 256 +print_info: n_embd_head_v = 256 +print_info: n_gqa = 6 +print_info: n_embd_k_gqa = 1024 +print_info: n_embd_v_gqa = 1024 +print_info: f_norm_eps = 0.0e+00 +print_info: f_norm_rms_eps = 1.0e-06 +print_info: f_clamp_kqv = 0.0e+00 +print_info: f_max_alibi_bias = 0.0e+00 +print_info: f_logit_scale = 0.0e+00 +print_info: f_attn_scale = 0.0e+00 +print_info: n_ff = 17408 +print_info: n_expert = 0 +print_info: n_expert_used = 0 +print_info: n_expert_groups = 0 +print_info: n_group_used = 0 +print_info: causal attn = 1 +print_info: pooling type = -1 +print_info: rope type = 40 +print_info: rope scaling = linear +print_info: freq_base_train = 10000000.0 +print_info: freq_scale_train = 1 +print_info: n_ctx_orig_yarn = 262144 +print_info: rope_yarn_log_mul = 0.0000 +print_info: rope_finetuned = unknown +print_info: mrope sections = [11, 11, 10, 0] +print_info: ssm_d_conv = 4 +print_info: ssm_d_inner = 6144 +print_info: ssm_d_state = 128 +print_info: ssm_dt_rank = 48 +print_info: ssm_n_group = 16 +print_info: ssm_dt_b_c_rms = 0 +print_info: model type = 27B +print_info: model params = 26.90 B +print_info: general.name = Qwen3.5-27B +print_info: vocab type = BPE +print_info: n_vocab = 248320 +print_info: n_merges = 247587 +print_info: BOS token = 11 ',' +print_info: EOS token = 248046 '<|im_end|>' +print_info: EOT token = 248046 '<|im_end|>' +print_info: PAD token = 248055 '<|vision_pad|>' +print_info: LF token = 198 'Ċ' +print_info: FIM PRE token = 248060 '<|fim_prefix|>' +print_info: FIM SUF token = 248062 '<|fim_suffix|>' +print_info: FIM MID token = 248061 '<|fim_middle|>' +print_info: FIM PAD token = 248063 '<|fim_pad|>' +print_info: FIM REP token = 248064 '<|repo_name|>' +print_info: FIM SEP token = 248065 '<|file_sep|>' +print_info: EOG token = 248044 '<|endoftext|>' +print_info: EOG token = 248046 '<|im_end|>' +print_info: EOG token = 248063 '<|fim_pad|>' +print_info: EOG token = 248064 '<|repo_name|>' +print_info: EOG token = 248065 '<|file_sep|>' +print_info: max token length = 256 +load_tensors: loading model tensors, this can take a while... (mmap = true, direct_io = false) +load_tensors: offloading output layer to GPU +load_tensors: offloading 63 repeating layers to GPU +load_tensors: offloaded 65/65 layers to GPU +load_tensors: CPU_Mapped model buffer size = 1288.28 MiB +load_tensors: MTL0_Mapped model buffer size = 27260.56 MiB +............................................................................................. +common_init_result: added <|endoftext|> logit bias = -inf +common_init_result: added <|im_end|> logit bias = -inf +common_init_result: added <|fim_pad|> logit bias = -inf +common_init_result: added <|repo_name|> logit bias = -inf +common_init_result: added <|file_sep|> logit bias = -inf +llama_context: constructing llama_context +llama_context: n_seq_max = 4 +llama_context: n_ctx = 2048 +llama_context: n_ctx_seq = 512 +llama_context: n_batch = 2048 +llama_context: n_ubatch = 512 +llama_context: causal_attn = 1 +llama_context: flash_attn = enabled +llama_context: kv_unified = false +llama_context: freq_base = 10000000.0 +llama_context: freq_scale = 1 +llama_context: n_ctx_seq (512) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +ggml_metal_init: allocating +ggml_metal_init: found device: Apple M5 Max +ggml_metal_init: picking default device: Apple M5 Max +ggml_metal_init: use fusion = true +ggml_metal_init: use concurrency = true +ggml_metal_init: use graph optimize = true +llama_context: CPU output buffer size = 3.79 MiB +llama_kv_cache: MTL0 KV buffer size = 68.00 MiB +llama_kv_cache: size = 68.00 MiB ( 512 cells, 16 layers, 4/4 seqs), K (q8_0): 34.00 MiB, V (q8_0): 34.00 MiB +llama_memory_recurrent: MTL0 RS buffer size = 598.50 MiB +llama_memory_recurrent: size = 598.50 MiB ( 4 cells, 64 layers, 4 seqs), R (f32): 22.50 MiB, S (f32): 576.00 MiB +sched_reserve: reserving ... +sched_reserve: resolving fused Gated Delta Net support: +sched_reserve: fused Gated Delta Net (autoregressive) enabled +sched_reserve: fused Gated Delta Net (chunked) enabled +sched_reserve: MTL0 compute buffer size = 495.00 MiB +sched_reserve: CPU compute buffer size = 21.02 MiB +sched_reserve: graph nodes = 3689 +sched_reserve: graph splits = 2 +sched_reserve: reserve took 24.55 ms, sched copies = 1 +common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) + +system_info: n_threads = 6 (n_threads_batch = 6) / 18 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | REPACK = 1 | +kl_divergence: computing over 8 chunks, n_ctx=512, batch_size=2048, n_seq=4 +kl_divergence: 4.36 seconds per pass - ETA 0.13 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 4.3889 ± 0.6289 0.00019 ± 0.00033 0.00001 ± 0.00000 0.070 ± 0.006 % 100.000 ± 0.000 % + 2 6.2073 ± 0.7277 0.00020 ± 0.00025 0.00001 ± 0.00000 0.089 ± 0.009 % 100.000 ± 0.000 % + 3 6.1931 ± 0.5947 0.00420 ± 0.00383 0.00001 ± 0.00000 0.094 ± 0.007 % 99.869 ± 0.131 % + 4 6.1925 ± 0.4950 0.00317 ± 0.00288 0.00002 ± 0.00001 0.161 ± 0.038 % 99.902 ± 0.098 % + 5 6.0978 ± 0.4209 0.00257 ± 0.00230 0.00002 ± 0.00001 0.149 ± 0.033 % 99.922 ± 0.078 % + 6 6.3059 ± 0.3964 0.00220 ± 0.00192 0.00002 ± 0.00000 0.139 ± 0.029 % 99.935 ± 0.065 % + 7 6.4256 ± 0.3712 0.00264 ± 0.00185 0.00002 ± 0.00000 0.133 ± 0.026 % 99.888 ± 0.079 % + 8 6.8884 ± 0.3822 0.00225 ± 0.00162 0.00002 ± 0.00000 0.127 ± 0.024 % 99.902 ± 0.069 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 6.888352 ± 0.382217 +Mean PPL(base) : 6.872893 ± 0.379370 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.96% +Mean ln(PPL(Q)/PPL(base)) : 0.002247 ± 0.001616 +Mean PPL(Q)/PPL(base) : 1.002249 ± 0.001620 +Mean PPL(Q)-PPL(base) : 0.015459 ± 0.011303 + +====== KL divergence statistics ====== +Mean KLD: 0.000018 ± 0.000004 +Maximum KLD: 0.006564 +99.9% KLD: 0.000212 +99.0% KLD: 0.000101 +95.0% KLD: 0.000051 +90.0% KLD: 0.000039 +Median KLD: 0.000007 +10.0% KLD: -0.000006 + 5.0% KLD: -0.000013 + 1.0% KLD: -0.000027 + 0.1% KLD: -0.000038 +Minimum KLD: -0.000040 + +====== Token probability statistics ====== +Mean Δp: -0.004 ± 0.003 % +Maximum Δp: 0.841% +99.9% Δp: 0.597% +99.0% Δp: 0.242% +95.0% Δp: 0.123% +90.0% Δp: 0.079% +75.0% Δp: 0.020% +Median Δp: 0.000% +25.0% Δp: -0.020% +10.0% Δp: -0.078% + 5.0% Δp: -0.138% + 1.0% Δp: -0.320% + 0.1% Δp: -0.611% +Minimum Δp: -3.256% +RMS Δp : 0.127 ± 0.024 % +Same top p: 99.902 ± 0.069 % + +llama_perf_context_print: load time = 1908.04 ms +llama_perf_context_print: prompt eval time = 4509.35 ms / 4096 tokens ( 1.10 ms per token, 908.34 tokens per second) +llama_perf_context_print: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) +llama_perf_context_print: total time = 9208.81 ms / 4097 tokens +llama_perf_context_print: graphs reused = 2 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - MTL0 (Apple M5 Max) | 110100 = 81677 + (28422 = 27260 + 666 + 495) + 0 | +llama_memory_breakdown_print: | - Host | 1309 = 1288 + 0 + 21 | +ggml_metal_free: deallocating diff --git a/benchmark-results-raw/kld_dense_turbo3.log b/benchmark-results-raw/kld_dense_turbo3.log new file mode 100644 index 0000000..0ef650e --- /dev/null +++ b/benchmark-results-raw/kld_dense_turbo3.log @@ -0,0 +1,271 @@ +ggml_metal_device_init: testing tensor API for f16 support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10527e1f0 | th_max = 1024 | th_width = 32 +ggml_metal_device_init: testing tensor API for bfloat support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10527cc70 | th_max = 1024 | th_width = 32 +ggml_metal_library_init: using embedded metal library +ggml_metal_library_init: turbo3 sparse V dequant enabled +ggml_metal_library_init: loaded in 0.008 sec +ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s) +ggml_metal_device_init: GPU name: MTL0 +ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010) +ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003) +ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002) +ggml_metal_device_init: simdgroup reduction = true +ggml_metal_device_init: simdgroup matrix mul. = true +ggml_metal_device_init: has unified memory = true +ggml_metal_device_init: has bfloat = true +ggml_metal_device_init: has tensor = true +ggml_metal_device_init: use residency sets = true +ggml_metal_device_init: use shared buffers = true +ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB +build: 159 (7d1bd95) with AppleClang 17.0.0.17000604 for Darwin arm64 +common_init_result: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on +llama_params_fit_impl: projected to use 27093 MiB of device memory vs. 109501 MiB of free device memory +llama_params_fit_impl: will leave 82407 >= 1024 MiB of free device memory, no changes needed +llama_params_fit: successfully fit params to free device memory +llama_params_fit: fitting params to free memory took 0.36 seconds +llama_model_load_from_file_impl: using device MTL0 (Apple M5 Max) (unknown id) - 110100 MiB free +llama_model_loader: loaded meta data with 49 key-value pairs and 851 tensors from /Users/tom/local_llms/models/Qwen3.5-27B-Q8_0.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35 +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +llama_model_loader: - kv 4: general.sampling.temp f32 = 0.600000 +llama_model_loader: - kv 5: general.name str = Qwen3.5-27B +llama_model_loader: - kv 6: general.basename str = Qwen3.5-27B +llama_model_loader: - kv 7: general.quantized_by str = Unsloth +llama_model_loader: - kv 8: general.size_label str = 27B +llama_model_loader: - kv 9: general.license str = apache-2.0 +llama_model_loader: - kv 10: general.license.link str = https://huggingface.co/Qwen/Qwen3.5-2... +llama_model_loader: - kv 11: general.repo_url str = https://huggingface.co/unsloth +llama_model_loader: - kv 12: general.base_model.count u32 = 1 +llama_model_loader: - kv 13: general.base_model.0.name str = Qwen3.5 27B +llama_model_loader: - kv 14: general.base_model.0.organization str = Qwen +llama_model_loader: - kv 15: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen3.5-27B +llama_model_loader: - kv 16: general.tags arr[str,3] = ["qwen3_5_moe", "unsloth", "image-tex... +llama_model_loader: - kv 17: qwen35.block_count u32 = 64 +llama_model_loader: - kv 18: qwen35.context_length u32 = 262144 +llama_model_loader: - kv 19: qwen35.embedding_length u32 = 5120 +llama_model_loader: - kv 20: qwen35.feed_forward_length u32 = 17408 +llama_model_loader: - kv 21: qwen35.attention.head_count u32 = 24 +llama_model_loader: - kv 22: qwen35.attention.head_count_kv u32 = 4 +llama_model_loader: - kv 23: qwen35.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 24: qwen35.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 25: qwen35.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 26: qwen35.attention.key_length u32 = 256 +llama_model_loader: - kv 27: qwen35.attention.value_length u32 = 256 +llama_model_loader: - kv 28: qwen35.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 29: qwen35.ssm.state_size u32 = 128 +llama_model_loader: - kv 30: qwen35.ssm.group_count u32 = 16 +llama_model_loader: - kv 31: qwen35.ssm.time_step_rank u32 = 48 +llama_model_loader: - kv 32: qwen35.ssm.inner_size u32 = 6144 +llama_model_loader: - kv 33: qwen35.full_attention_interval u32 = 4 +llama_model_loader: - kv 34: qwen35.rope.dimension_count u32 = 64 +llama_model_loader: - kv 35: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 36: tokenizer.ggml.pre str = qwen35 +llama_model_loader: - kv 37: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 38: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 39: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 40: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 41: tokenizer.ggml.padding_token_id u32 = 248055 +llama_model_loader: - kv 42: tokenizer.chat_template str = {%- set image_count = namespace(value... +llama_model_loader: - kv 43: general.quantization_version u32 = 2 +llama_model_loader: - kv 44: general.file_type u32 = 7 +llama_model_loader: - kv 45: quantize.imatrix.file str = Qwen3.5-27B-GGUF/imatrix_unsloth.gguf +llama_model_loader: - kv 46: quantize.imatrix.dataset str = unsloth_calibration_Qwen3.5-27B.txt +llama_model_loader: - kv 47: quantize.imatrix.entries_count u32 = 496 +llama_model_loader: - kv 48: quantize.imatrix.chunks_count u32 = 80 +llama_model_loader: - type f32: 353 tensors +llama_model_loader: - type q8_0: 498 tensors +print_info: file format = GGUF V3 (latest) +print_info: file type = Q8_0 +print_info: file size = 26.62 GiB (8.50 BPW) +load: 0 unused tokens +load: printing all EOG tokens: +load: - 248044 ('<|endoftext|>') +load: - 248046 ('<|im_end|>') +load: - 248063 ('<|fim_pad|>') +load: - 248064 ('<|repo_name|>') +load: - 248065 ('<|file_sep|>') +load: special tokens cache size = 33 +load: token to piece cache size = 1.7581 MB +print_info: arch = qwen35 +print_info: vocab_only = 0 +print_info: no_alloc = 0 +print_info: n_ctx_train = 262144 +print_info: n_embd = 5120 +print_info: n_embd_inp = 5120 +print_info: n_layer = 64 +print_info: n_head = 24 +print_info: n_head_kv = 4 +print_info: n_rot = 64 +print_info: n_swa = 0 +print_info: is_swa_any = 0 +print_info: n_embd_head_k = 256 +print_info: n_embd_head_v = 256 +print_info: n_gqa = 6 +print_info: n_embd_k_gqa = 1024 +print_info: n_embd_v_gqa = 1024 +print_info: f_norm_eps = 0.0e+00 +print_info: f_norm_rms_eps = 1.0e-06 +print_info: f_clamp_kqv = 0.0e+00 +print_info: f_max_alibi_bias = 0.0e+00 +print_info: f_logit_scale = 0.0e+00 +print_info: f_attn_scale = 0.0e+00 +print_info: n_ff = 17408 +print_info: n_expert = 0 +print_info: n_expert_used = 0 +print_info: n_expert_groups = 0 +print_info: n_group_used = 0 +print_info: causal attn = 1 +print_info: pooling type = -1 +print_info: rope type = 40 +print_info: rope scaling = linear +print_info: freq_base_train = 10000000.0 +print_info: freq_scale_train = 1 +print_info: n_ctx_orig_yarn = 262144 +print_info: rope_yarn_log_mul = 0.0000 +print_info: rope_finetuned = unknown +print_info: mrope sections = [11, 11, 10, 0] +print_info: ssm_d_conv = 4 +print_info: ssm_d_inner = 6144 +print_info: ssm_d_state = 128 +print_info: ssm_dt_rank = 48 +print_info: ssm_n_group = 16 +print_info: ssm_dt_b_c_rms = 0 +print_info: model type = 27B +print_info: model params = 26.90 B +print_info: general.name = Qwen3.5-27B +print_info: vocab type = BPE +print_info: n_vocab = 248320 +print_info: n_merges = 247587 +print_info: BOS token = 11 ',' +print_info: EOS token = 248046 '<|im_end|>' +print_info: EOT token = 248046 '<|im_end|>' +print_info: PAD token = 248055 '<|vision_pad|>' +print_info: LF token = 198 'Ċ' +print_info: FIM PRE token = 248060 '<|fim_prefix|>' +print_info: FIM SUF token = 248062 '<|fim_suffix|>' +print_info: FIM MID token = 248061 '<|fim_middle|>' +print_info: FIM PAD token = 248063 '<|fim_pad|>' +print_info: FIM REP token = 248064 '<|repo_name|>' +print_info: FIM SEP token = 248065 '<|file_sep|>' +print_info: EOG token = 248044 '<|endoftext|>' +print_info: EOG token = 248046 '<|im_end|>' +print_info: EOG token = 248063 '<|fim_pad|>' +print_info: EOG token = 248064 '<|repo_name|>' +print_info: EOG token = 248065 '<|file_sep|>' +print_info: max token length = 256 +load_tensors: loading model tensors, this can take a while... (mmap = true, direct_io = false) +load_tensors: offloading output layer to GPU +load_tensors: offloading 63 repeating layers to GPU +load_tensors: offloaded 65/65 layers to GPU +load_tensors: CPU_Mapped model buffer size = 1288.28 MiB +load_tensors: MTL0_Mapped model buffer size = 27260.56 MiB +............................................................................................. +common_init_result: added <|endoftext|> logit bias = -inf +common_init_result: added <|im_end|> logit bias = -inf +common_init_result: added <|fim_pad|> logit bias = -inf +common_init_result: added <|repo_name|> logit bias = -inf +common_init_result: added <|file_sep|> logit bias = -inf +llama_context: constructing llama_context +llama_context: n_seq_max = 4 +llama_context: n_ctx = 2048 +llama_context: n_ctx_seq = 512 +llama_context: n_batch = 2048 +llama_context: n_ubatch = 512 +llama_context: causal_attn = 1 +llama_context: flash_attn = enabled +llama_context: kv_unified = false +llama_context: freq_base = 10000000.0 +llama_context: freq_scale = 1 +llama_context: n_ctx_seq (512) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +ggml_metal_init: allocating +ggml_metal_init: found device: Apple M5 Max +ggml_metal_init: picking default device: Apple M5 Max +ggml_metal_init: use fusion = true +ggml_metal_init: use concurrency = true +ggml_metal_init: use graph optimize = true +llama_context: CPU output buffer size = 3.79 MiB +llama_kv_cache: MTL0 KV buffer size = 28.12 MiB +llama_kv_cache: TurboQuant rotation matrices initialized (128x128) +llama_kv_cache: size = 28.00 MiB ( 512 cells, 16 layers, 4/4 seqs), K (turbo3): 14.00 MiB, V (turbo3): 14.00 MiB +llama_memory_recurrent: MTL0 RS buffer size = 598.50 MiB +llama_memory_recurrent: size = 598.50 MiB ( 4 cells, 64 layers, 4 seqs), R (f32): 22.50 MiB, S (f32): 576.00 MiB +sched_reserve: reserving ... +sched_reserve: resolving fused Gated Delta Net support: +sched_reserve: fused Gated Delta Net (autoregressive) enabled +sched_reserve: fused Gated Delta Net (chunked) enabled +sched_reserve: MTL0 compute buffer size = 495.00 MiB +sched_reserve: CPU compute buffer size = 21.02 MiB +sched_reserve: graph nodes = 3721 +sched_reserve: graph splits = 2 +sched_reserve: reserve took 24.66 ms, sched copies = 1 +common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) + +system_info: n_threads = 6 (n_threads_batch = 6) / 18 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | REPACK = 1 | +kl_divergence: computing over 8 chunks, n_ctx=512, batch_size=2048, n_seq=4 +kl_divergence: 4.28 seconds per pass - ETA 0.13 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 4.3891 ± 0.6285 0.00023 ± 0.00865 0.01188 ± 0.00508 2.850 ± 0.644 % 96.078 ± 1.218 % + 2 6.2904 ± 0.7414 0.01348 ± 0.00709 0.01128 ± 0.00258 2.789 ± 0.366 % 96.275 ± 0.839 % + 3 6.3143 ± 0.6098 0.02357 ± 0.00670 0.01117 ± 0.00178 2.908 ± 0.264 % 96.078 ± 0.702 % + 4 6.3391 ± 0.5110 0.02657 ± 0.00608 0.01129 ± 0.00143 2.992 ± 0.228 % 95.686 ± 0.636 % + 5 6.2373 ± 0.4343 0.02519 ± 0.00517 0.01070 ± 0.00117 2.887 ± 0.193 % 95.765 ± 0.564 % + 6 6.4284 ± 0.4061 0.02143 ± 0.00454 0.01002 ± 0.00098 2.769 ± 0.170 % 95.882 ± 0.508 % + 7 6.5377 ± 0.3789 0.01993 ± 0.00413 0.00973 ± 0.00085 2.823 ± 0.166 % 96.022 ± 0.463 % + 8 7.0066 ± 0.3904 0.01927 ± 0.00378 0.00990 ± 0.00087 2.738 ± 0.152 % 95.980 ± 0.435 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 7.006588 ± 0.390413 +Mean PPL(base) : 6.872893 ± 0.379370 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.77% +Mean ln(PPL(Q)/PPL(base)) : 0.019266 ± 0.003778 +Mean PPL(Q)/PPL(base) : 1.019453 ± 0.003851 +Mean PPL(Q)-PPL(base) : 0.133695 ± 0.028215 + +====== KL divergence statistics ====== +Mean KLD: 0.009900 ± 0.000868 +Maximum KLD: 1.290557 +99.9% KLD: 0.380763 +99.0% KLD: 0.074194 +95.0% KLD: 0.027761 +90.0% KLD: 0.018358 +Median KLD: 0.004675 +10.0% KLD: 0.000291 + 5.0% KLD: 0.000093 + 1.0% KLD: 0.000008 + 0.1% KLD: 0.000000 +Minimum KLD: -0.000000 + +====== Token probability statistics ====== +Mean Δp: -0.237 ± 0.060 % +Maximum Δp: 30.386% +99.9% Δp: 16.228% +99.0% Δp: 7.073% +95.0% Δp: 3.093% +90.0% Δp: 1.901% +75.0% Δp: 0.384% +Median Δp: -0.013% +25.0% Δp: -0.743% +10.0% Δp: -2.420% + 5.0% Δp: -4.054% + 1.0% Δp: -9.246% + 0.1% Δp: -21.460% +Minimum Δp: -23.586% +RMS Δp : 2.738 ± 0.152 % +Same top p: 95.980 ± 0.435 % + +llama_perf_context_print: load time = 1900.16 ms +llama_perf_context_print: prompt eval time = 4950.38 ms / 4096 tokens ( 1.21 ms per token, 827.41 tokens per second) +llama_perf_context_print: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) +llama_perf_context_print: total time = 9575.86 ms / 4097 tokens +llama_perf_context_print: graphs reused = 2 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - MTL0 (Apple M5 Max) | 110100 = 81717 + (28382 = 27260 + 626 + 495) + 0 | +llama_memory_breakdown_print: | - Host | 1309 = 1288 + 0 + 21 | +ggml_metal_free: deallocating diff --git a/benchmark-results-raw/kld_moe_q4_0.log b/benchmark-results-raw/kld_moe_q4_0.log new file mode 100644 index 0000000..edd336f --- /dev/null +++ b/benchmark-results-raw/kld_moe_q4_0.log @@ -0,0 +1,273 @@ +ggml_metal_device_init: testing tensor API for f16 support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1018627f0 | th_max = 1024 | th_width = 32 +ggml_metal_device_init: testing tensor API for bfloat support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x101862470 | th_max = 1024 | th_width = 32 +ggml_metal_library_init: using embedded metal library +ggml_metal_library_init: turbo3 sparse V dequant enabled +ggml_metal_library_init: loaded in 0.009 sec +ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s) +ggml_metal_device_init: GPU name: MTL0 +ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010) +ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003) +ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002) +ggml_metal_device_init: simdgroup reduction = true +ggml_metal_device_init: simdgroup matrix mul. = true +ggml_metal_device_init: has unified memory = true +ggml_metal_device_init: has bfloat = true +ggml_metal_device_init: has tensor = true +ggml_metal_device_init: use residency sets = true +ggml_metal_device_init: use shared buffers = true +ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB +build: 159 (7d1bd95) with AppleClang 17.0.0.17000604 for Darwin arm64 +common_init_result: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on +llama_params_fit_impl: projected to use 35423 MiB of device memory vs. 109848 MiB of free device memory +llama_params_fit_impl: will leave 74425 >= 1024 MiB of free device memory, no changes needed +llama_params_fit: successfully fit params to free device memory +llama_params_fit: fitting params to free memory took 0.25 seconds +llama_model_load_from_file_impl: using device MTL0 (Apple M5 Max) (unknown id) - 110100 MiB free +llama_model_loader: loaded meta data with 52 key-value pairs and 733 tensors from /Users/tom/local_llms/models/Qwen3.5-35B-A3B-Q8_0.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35moe +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +llama_model_loader: - kv 5: general.name str = Qwen3.5-35B-A3B +llama_model_loader: - kv 6: general.basename str = Qwen3.5-35B-A3B +llama_model_loader: - kv 7: general.quantized_by str = Unsloth +llama_model_loader: - kv 8: general.size_label str = 35B-A3B +llama_model_loader: - kv 9: general.license str = apache-2.0 +llama_model_loader: - kv 10: general.license.link str = https://huggingface.co/Qwen/Qwen3.5-3... +llama_model_loader: - kv 11: general.repo_url str = https://huggingface.co/unsloth +llama_model_loader: - kv 12: general.base_model.count u32 = 1 +llama_model_loader: - kv 13: general.base_model.0.name str = Qwen3.5 35B A3B +llama_model_loader: - kv 14: general.base_model.0.organization str = Qwen +llama_model_loader: - kv 15: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen3.5-3... +llama_model_loader: - kv 16: general.tags arr[str,2] = ["unsloth", "image-text-to-text"] +llama_model_loader: - kv 17: qwen35moe.block_count u32 = 40 +llama_model_loader: - kv 18: qwen35moe.context_length u32 = 262144 +llama_model_loader: - kv 19: qwen35moe.embedding_length u32 = 2048 +llama_model_loader: - kv 20: qwen35moe.attention.head_count u32 = 16 +llama_model_loader: - kv 21: qwen35moe.attention.head_count_kv u32 = 2 +llama_model_loader: - kv 22: qwen35moe.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 23: qwen35moe.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 24: qwen35moe.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 25: qwen35moe.expert_count u32 = 256 +llama_model_loader: - kv 26: qwen35moe.expert_used_count u32 = 8 +llama_model_loader: - kv 27: qwen35moe.attention.key_length u32 = 256 +llama_model_loader: - kv 28: qwen35moe.attention.value_length u32 = 256 +llama_model_loader: - kv 29: qwen35moe.expert_feed_forward_length u32 = 512 +llama_model_loader: - kv 30: qwen35moe.expert_shared_feed_forward_length u32 = 512 +llama_model_loader: - kv 31: qwen35moe.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 32: qwen35moe.ssm.state_size u32 = 128 +llama_model_loader: - kv 33: qwen35moe.ssm.group_count u32 = 16 +llama_model_loader: - kv 34: qwen35moe.ssm.time_step_rank u32 = 32 +llama_model_loader: - kv 35: qwen35moe.ssm.inner_size u32 = 4096 +llama_model_loader: - kv 36: qwen35moe.full_attention_interval u32 = 4 +llama_model_loader: - kv 37: qwen35moe.rope.dimension_count u32 = 64 +llama_model_loader: - kv 38: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 39: tokenizer.ggml.pre str = qwen35 +llama_model_loader: - kv 40: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 41: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 42: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 43: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 44: tokenizer.ggml.padding_token_id u32 = 248055 +llama_model_loader: - kv 45: tokenizer.chat_template str = {%- set image_count = namespace(value... +llama_model_loader: - kv 46: general.quantization_version u32 = 2 +llama_model_loader: - kv 47: general.file_type u32 = 7 +llama_model_loader: - kv 48: quantize.imatrix.file str = Qwen3.5-35B-A3B-GGUF/imatrix_unsloth.... +llama_model_loader: - kv 49: quantize.imatrix.dataset str = unsloth_calibration_Qwen3.5-35B-A3B.txt +llama_model_loader: - kv 50: quantize.imatrix.entries_count u32 = 510 +llama_model_loader: - kv 51: quantize.imatrix.chunks_count u32 = 76 +llama_model_loader: - type f32: 301 tensors +llama_model_loader: - type q8_0: 432 tensors +print_info: file format = GGUF V3 (latest) +print_info: file type = Q8_0 +print_info: file size = 34.36 GiB (8.52 BPW) +load: 0 unused tokens +load: printing all EOG tokens: +load: - 248044 ('<|endoftext|>') +load: - 248046 ('<|im_end|>') +load: - 248063 ('<|fim_pad|>') +load: - 248064 ('<|repo_name|>') +load: - 248065 ('<|file_sep|>') +load: special tokens cache size = 33 +load: token to piece cache size = 1.7581 MB +print_info: arch = qwen35moe +print_info: vocab_only = 0 +print_info: no_alloc = 0 +print_info: n_ctx_train = 262144 +print_info: n_embd = 2048 +print_info: n_embd_inp = 2048 +print_info: n_layer = 40 +print_info: n_head = 16 +print_info: n_head_kv = 2 +print_info: n_rot = 64 +print_info: n_swa = 0 +print_info: is_swa_any = 0 +print_info: n_embd_head_k = 256 +print_info: n_embd_head_v = 256 +print_info: n_gqa = 8 +print_info: n_embd_k_gqa = 512 +print_info: n_embd_v_gqa = 512 +print_info: f_norm_eps = 0.0e+00 +print_info: f_norm_rms_eps = 1.0e-06 +print_info: f_clamp_kqv = 0.0e+00 +print_info: f_max_alibi_bias = 0.0e+00 +print_info: f_logit_scale = 0.0e+00 +print_info: f_attn_scale = 0.0e+00 +print_info: n_ff = 0 +print_info: n_expert = 256 +print_info: n_expert_used = 8 +print_info: n_expert_groups = 0 +print_info: n_group_used = 0 +print_info: causal attn = 1 +print_info: pooling type = -1 +print_info: rope type = 40 +print_info: rope scaling = linear +print_info: freq_base_train = 10000000.0 +print_info: freq_scale_train = 1 +print_info: n_ctx_orig_yarn = 262144 +print_info: rope_yarn_log_mul = 0.0000 +print_info: rope_finetuned = unknown +print_info: mrope sections = [11, 11, 10, 0] +print_info: ssm_d_conv = 4 +print_info: ssm_d_inner = 4096 +print_info: ssm_d_state = 128 +print_info: ssm_dt_rank = 32 +print_info: ssm_n_group = 16 +print_info: ssm_dt_b_c_rms = 0 +print_info: model type = 35B.A3B +print_info: model params = 34.66 B +print_info: general.name = Qwen3.5-35B-A3B +print_info: vocab type = BPE +print_info: n_vocab = 248320 +print_info: n_merges = 247587 +print_info: BOS token = 11 ',' +print_info: EOS token = 248046 '<|im_end|>' +print_info: EOT token = 248046 '<|im_end|>' +print_info: PAD token = 248055 '<|vision_pad|>' +print_info: LF token = 198 'Ċ' +print_info: FIM PRE token = 248060 '<|fim_prefix|>' +print_info: FIM SUF token = 248062 '<|fim_suffix|>' +print_info: FIM MID token = 248061 '<|fim_middle|>' +print_info: FIM PAD token = 248063 '<|fim_pad|>' +print_info: FIM REP token = 248064 '<|repo_name|>' +print_info: FIM SEP token = 248065 '<|file_sep|>' +print_info: EOG token = 248044 '<|endoftext|>' +print_info: EOG token = 248046 '<|im_end|>' +print_info: EOG token = 248063 '<|fim_pad|>' +print_info: EOG token = 248064 '<|repo_name|>' +print_info: EOG token = 248065 '<|file_sep|>' +print_info: max token length = 256 +load_tensors: loading model tensors, this can take a while... (mmap = true, direct_io = false) +load_tensors: offloading output layer to GPU +load_tensors: offloading 39 repeating layers to GPU +load_tensors: offloaded 41/41 layers to GPU +load_tensors: CPU_Mapped model buffer size = 515.31 MiB +load_tensors: MTL0_Mapped model buffer size = 35183.10 MiB +.................................................................................................... +common_init_result: added <|endoftext|> logit bias = -inf +common_init_result: added <|im_end|> logit bias = -inf +common_init_result: added <|fim_pad|> logit bias = -inf +common_init_result: added <|repo_name|> logit bias = -inf +common_init_result: added <|file_sep|> logit bias = -inf +llama_context: constructing llama_context +llama_context: n_seq_max = 4 +llama_context: n_ctx = 2048 +llama_context: n_ctx_seq = 512 +llama_context: n_batch = 2048 +llama_context: n_ubatch = 512 +llama_context: causal_attn = 1 +llama_context: flash_attn = enabled +llama_context: kv_unified = false +llama_context: freq_base = 10000000.0 +llama_context: freq_scale = 1 +llama_context: n_ctx_seq (512) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +ggml_metal_init: allocating +ggml_metal_init: found device: Apple M5 Max +ggml_metal_init: picking default device: Apple M5 Max +ggml_metal_init: use fusion = true +ggml_metal_init: use concurrency = true +ggml_metal_init: use graph optimize = true +llama_context: CPU output buffer size = 3.79 MiB +llama_kv_cache: MTL0 KV buffer size = 11.25 MiB +llama_kv_cache: size = 11.25 MiB ( 512 cells, 10 layers, 4/4 seqs), K (q4_0): 5.62 MiB, V (q4_0): 5.62 MiB +llama_memory_recurrent: MTL0 RS buffer size = 251.25 MiB +llama_memory_recurrent: size = 251.25 MiB ( 4 cells, 40 layers, 4 seqs), R (f32): 11.25 MiB, S (f32): 240.00 MiB +sched_reserve: reserving ... +sched_reserve: resolving fused Gated Delta Net support: +sched_reserve: fused Gated Delta Net (autoregressive) enabled +sched_reserve: fused Gated Delta Net (chunked) enabled +sched_reserve: MTL0 compute buffer size = 493.00 MiB +sched_reserve: CPU compute buffer size = 9.02 MiB +sched_reserve: graph nodes = 3749 +sched_reserve: graph splits = 2 +sched_reserve: reserve took 22.20 ms, sched copies = 1 +common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) + +system_info: n_threads = 6 (n_threads_batch = 6) / 18 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | REPACK = 1 | +kl_divergence: computing over 8 chunks, n_ctx=512, batch_size=2048, n_seq=4 +kl_divergence: 0.94 seconds per pass - ETA 0.02 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 4.2535 ± 0.5938 0.01064 ± 0.00834 0.00602 ± 0.00075 2.204 ± 0.222 % 98.039 ± 0.870 % + 2 6.0305 ± 0.7005 0.00343 ± 0.00550 0.00678 ± 0.00053 2.319 ± 0.226 % 96.275 ± 0.839 % + 3 5.1194 ± 0.4644 0.00627 ± 0.00491 0.00836 ± 0.00051 2.896 ± 0.209 % 96.209 ± 0.691 % + 4 5.0906 ± 0.3783 0.00346 ± 0.00412 0.00791 ± 0.00041 2.788 ± 0.175 % 95.784 ± 0.629 % + 5 5.2499 ± 0.3424 0.00199 ± 0.00372 0.00799 ± 0.00039 2.847 ± 0.186 % 95.686 ± 0.569 % + 6 5.4190 ± 0.3215 0.00031 ± 0.00338 0.00797 ± 0.00034 2.801 ± 0.163 % 95.621 ± 0.523 % + 7 5.6227 ± 0.3100 0.00333 ± 0.00339 0.00798 ± 0.00031 2.715 ± 0.146 % 95.742 ± 0.478 % + 8 6.1424 ± 0.3270 0.00436 ± 0.00313 0.00809 ± 0.00029 2.753 ± 0.138 % 95.833 ± 0.443 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 6.142449 ± 0.327049 +Mean PPL(base) : 6.115704 ± 0.325284 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.83% +Mean ln(PPL(Q)/PPL(base)) : 0.004364 ± 0.003129 +Mean PPL(Q)/PPL(base) : 1.004373 ± 0.003143 +Mean PPL(Q)-PPL(base) : 0.026745 ± 0.019256 + +====== KL divergence statistics ====== +Mean KLD: 0.008091 ± 0.000292 +Maximum KLD: 0.228670 +99.9% KLD: 0.133784 +99.0% KLD: 0.063379 +95.0% KLD: 0.027492 +90.0% KLD: 0.018377 +Median KLD: 0.004403 +10.0% KLD: 0.000252 + 5.0% KLD: 0.000082 + 1.0% KLD: 0.000015 + 0.1% KLD: 0.000002 +Minimum KLD: -0.000002 + +====== Token probability statistics ====== +Mean Δp: -0.171 ± 0.061 % +Maximum Δp: 20.505% +99.9% Δp: 16.916% +99.0% Δp: 8.096% +95.0% Δp: 3.384% +90.0% Δp: 2.151% +75.0% Δp: 0.464% +Median Δp: -0.008% +25.0% Δp: -0.737% +10.0% Δp: -2.572% + 5.0% Δp: -4.112% + 1.0% Δp: -9.084% + 0.1% Δp: -19.815% +Minimum Δp: -29.275% +RMS Δp : 2.753 ± 0.138 % +Same top p: 95.833 ± 0.443 % + +llama_perf_context_print: load time = 2306.32 ms +llama_perf_context_print: prompt eval time = 843.12 ms / 4096 tokens ( 0.21 ms per token, 4858.17 tokens per second) +llama_perf_context_print: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) +llama_perf_context_print: total time = 2147.86 ms / 4097 tokens +llama_perf_context_print: graphs reused = 2 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - MTL0 (Apple M5 Max) | 110100 = 74161 + (35938 = 35183 + 262 + 493) + 0 | +llama_memory_breakdown_print: | - Host | 524 = 515 + 0 + 9 | +ggml_metal_free: deallocating diff --git a/benchmark-results-raw/kld_moe_q8_0.log b/benchmark-results-raw/kld_moe_q8_0.log new file mode 100644 index 0000000..0986d53 --- /dev/null +++ b/benchmark-results-raw/kld_moe_q8_0.log @@ -0,0 +1,273 @@ +ggml_metal_device_init: testing tensor API for f16 support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10655a080 | th_max = 1024 | th_width = 32 +ggml_metal_device_init: testing tensor API for bfloat support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x106559d00 | th_max = 1024 | th_width = 32 +ggml_metal_library_init: using embedded metal library +ggml_metal_library_init: turbo3 sparse V dequant enabled +ggml_metal_library_init: loaded in 0.007 sec +ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s) +ggml_metal_device_init: GPU name: MTL0 +ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010) +ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003) +ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002) +ggml_metal_device_init: simdgroup reduction = true +ggml_metal_device_init: simdgroup matrix mul. = true +ggml_metal_device_init: has unified memory = true +ggml_metal_device_init: has bfloat = true +ggml_metal_device_init: has tensor = true +ggml_metal_device_init: use residency sets = true +ggml_metal_device_init: use shared buffers = true +ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB +build: 159 (7d1bd95) with AppleClang 17.0.0.17000604 for Darwin arm64 +common_init_result: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on +llama_params_fit_impl: projected to use 35433 MiB of device memory vs. 109848 MiB of free device memory +llama_params_fit_impl: will leave 74415 >= 1024 MiB of free device memory, no changes needed +llama_params_fit: successfully fit params to free device memory +llama_params_fit: fitting params to free memory took 0.25 seconds +llama_model_load_from_file_impl: using device MTL0 (Apple M5 Max) (unknown id) - 110100 MiB free +llama_model_loader: loaded meta data with 52 key-value pairs and 733 tensors from /Users/tom/local_llms/models/Qwen3.5-35B-A3B-Q8_0.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35moe +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +llama_model_loader: - kv 5: general.name str = Qwen3.5-35B-A3B +llama_model_loader: - kv 6: general.basename str = Qwen3.5-35B-A3B +llama_model_loader: - kv 7: general.quantized_by str = Unsloth +llama_model_loader: - kv 8: general.size_label str = 35B-A3B +llama_model_loader: - kv 9: general.license str = apache-2.0 +llama_model_loader: - kv 10: general.license.link str = https://huggingface.co/Qwen/Qwen3.5-3... +llama_model_loader: - kv 11: general.repo_url str = https://huggingface.co/unsloth +llama_model_loader: - kv 12: general.base_model.count u32 = 1 +llama_model_loader: - kv 13: general.base_model.0.name str = Qwen3.5 35B A3B +llama_model_loader: - kv 14: general.base_model.0.organization str = Qwen +llama_model_loader: - kv 15: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen3.5-3... +llama_model_loader: - kv 16: general.tags arr[str,2] = ["unsloth", "image-text-to-text"] +llama_model_loader: - kv 17: qwen35moe.block_count u32 = 40 +llama_model_loader: - kv 18: qwen35moe.context_length u32 = 262144 +llama_model_loader: - kv 19: qwen35moe.embedding_length u32 = 2048 +llama_model_loader: - kv 20: qwen35moe.attention.head_count u32 = 16 +llama_model_loader: - kv 21: qwen35moe.attention.head_count_kv u32 = 2 +llama_model_loader: - kv 22: qwen35moe.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 23: qwen35moe.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 24: qwen35moe.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 25: qwen35moe.expert_count u32 = 256 +llama_model_loader: - kv 26: qwen35moe.expert_used_count u32 = 8 +llama_model_loader: - kv 27: qwen35moe.attention.key_length u32 = 256 +llama_model_loader: - kv 28: qwen35moe.attention.value_length u32 = 256 +llama_model_loader: - kv 29: qwen35moe.expert_feed_forward_length u32 = 512 +llama_model_loader: - kv 30: qwen35moe.expert_shared_feed_forward_length u32 = 512 +llama_model_loader: - kv 31: qwen35moe.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 32: qwen35moe.ssm.state_size u32 = 128 +llama_model_loader: - kv 33: qwen35moe.ssm.group_count u32 = 16 +llama_model_loader: - kv 34: qwen35moe.ssm.time_step_rank u32 = 32 +llama_model_loader: - kv 35: qwen35moe.ssm.inner_size u32 = 4096 +llama_model_loader: - kv 36: qwen35moe.full_attention_interval u32 = 4 +llama_model_loader: - kv 37: qwen35moe.rope.dimension_count u32 = 64 +llama_model_loader: - kv 38: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 39: tokenizer.ggml.pre str = qwen35 +llama_model_loader: - kv 40: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 41: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 42: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 43: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 44: tokenizer.ggml.padding_token_id u32 = 248055 +llama_model_loader: - kv 45: tokenizer.chat_template str = {%- set image_count = namespace(value... +llama_model_loader: - kv 46: general.quantization_version u32 = 2 +llama_model_loader: - kv 47: general.file_type u32 = 7 +llama_model_loader: - kv 48: quantize.imatrix.file str = Qwen3.5-35B-A3B-GGUF/imatrix_unsloth.... +llama_model_loader: - kv 49: quantize.imatrix.dataset str = unsloth_calibration_Qwen3.5-35B-A3B.txt +llama_model_loader: - kv 50: quantize.imatrix.entries_count u32 = 510 +llama_model_loader: - kv 51: quantize.imatrix.chunks_count u32 = 76 +llama_model_loader: - type f32: 301 tensors +llama_model_loader: - type q8_0: 432 tensors +print_info: file format = GGUF V3 (latest) +print_info: file type = Q8_0 +print_info: file size = 34.36 GiB (8.52 BPW) +load: 0 unused tokens +load: printing all EOG tokens: +load: - 248044 ('<|endoftext|>') +load: - 248046 ('<|im_end|>') +load: - 248063 ('<|fim_pad|>') +load: - 248064 ('<|repo_name|>') +load: - 248065 ('<|file_sep|>') +load: special tokens cache size = 33 +load: token to piece cache size = 1.7581 MB +print_info: arch = qwen35moe +print_info: vocab_only = 0 +print_info: no_alloc = 0 +print_info: n_ctx_train = 262144 +print_info: n_embd = 2048 +print_info: n_embd_inp = 2048 +print_info: n_layer = 40 +print_info: n_head = 16 +print_info: n_head_kv = 2 +print_info: n_rot = 64 +print_info: n_swa = 0 +print_info: is_swa_any = 0 +print_info: n_embd_head_k = 256 +print_info: n_embd_head_v = 256 +print_info: n_gqa = 8 +print_info: n_embd_k_gqa = 512 +print_info: n_embd_v_gqa = 512 +print_info: f_norm_eps = 0.0e+00 +print_info: f_norm_rms_eps = 1.0e-06 +print_info: f_clamp_kqv = 0.0e+00 +print_info: f_max_alibi_bias = 0.0e+00 +print_info: f_logit_scale = 0.0e+00 +print_info: f_attn_scale = 0.0e+00 +print_info: n_ff = 0 +print_info: n_expert = 256 +print_info: n_expert_used = 8 +print_info: n_expert_groups = 0 +print_info: n_group_used = 0 +print_info: causal attn = 1 +print_info: pooling type = -1 +print_info: rope type = 40 +print_info: rope scaling = linear +print_info: freq_base_train = 10000000.0 +print_info: freq_scale_train = 1 +print_info: n_ctx_orig_yarn = 262144 +print_info: rope_yarn_log_mul = 0.0000 +print_info: rope_finetuned = unknown +print_info: mrope sections = [11, 11, 10, 0] +print_info: ssm_d_conv = 4 +print_info: ssm_d_inner = 4096 +print_info: ssm_d_state = 128 +print_info: ssm_dt_rank = 32 +print_info: ssm_n_group = 16 +print_info: ssm_dt_b_c_rms = 0 +print_info: model type = 35B.A3B +print_info: model params = 34.66 B +print_info: general.name = Qwen3.5-35B-A3B +print_info: vocab type = BPE +print_info: n_vocab = 248320 +print_info: n_merges = 247587 +print_info: BOS token = 11 ',' +print_info: EOS token = 248046 '<|im_end|>' +print_info: EOT token = 248046 '<|im_end|>' +print_info: PAD token = 248055 '<|vision_pad|>' +print_info: LF token = 198 'Ċ' +print_info: FIM PRE token = 248060 '<|fim_prefix|>' +print_info: FIM SUF token = 248062 '<|fim_suffix|>' +print_info: FIM MID token = 248061 '<|fim_middle|>' +print_info: FIM PAD token = 248063 '<|fim_pad|>' +print_info: FIM REP token = 248064 '<|repo_name|>' +print_info: FIM SEP token = 248065 '<|file_sep|>' +print_info: EOG token = 248044 '<|endoftext|>' +print_info: EOG token = 248046 '<|im_end|>' +print_info: EOG token = 248063 '<|fim_pad|>' +print_info: EOG token = 248064 '<|repo_name|>' +print_info: EOG token = 248065 '<|file_sep|>' +print_info: max token length = 256 +load_tensors: loading model tensors, this can take a while... (mmap = true, direct_io = false) +load_tensors: offloading output layer to GPU +load_tensors: offloading 39 repeating layers to GPU +load_tensors: offloaded 41/41 layers to GPU +load_tensors: CPU_Mapped model buffer size = 515.31 MiB +load_tensors: MTL0_Mapped model buffer size = 35183.10 MiB +.................................................................................................... +common_init_result: added <|endoftext|> logit bias = -inf +common_init_result: added <|im_end|> logit bias = -inf +common_init_result: added <|fim_pad|> logit bias = -inf +common_init_result: added <|repo_name|> logit bias = -inf +common_init_result: added <|file_sep|> logit bias = -inf +llama_context: constructing llama_context +llama_context: n_seq_max = 4 +llama_context: n_ctx = 2048 +llama_context: n_ctx_seq = 512 +llama_context: n_batch = 2048 +llama_context: n_ubatch = 512 +llama_context: causal_attn = 1 +llama_context: flash_attn = enabled +llama_context: kv_unified = false +llama_context: freq_base = 10000000.0 +llama_context: freq_scale = 1 +llama_context: n_ctx_seq (512) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +ggml_metal_init: allocating +ggml_metal_init: found device: Apple M5 Max +ggml_metal_init: picking default device: Apple M5 Max +ggml_metal_init: use fusion = true +ggml_metal_init: use concurrency = true +ggml_metal_init: use graph optimize = true +llama_context: CPU output buffer size = 3.79 MiB +llama_kv_cache: MTL0 KV buffer size = 21.25 MiB +llama_kv_cache: size = 21.25 MiB ( 512 cells, 10 layers, 4/4 seqs), K (q8_0): 10.62 MiB, V (q8_0): 10.62 MiB +llama_memory_recurrent: MTL0 RS buffer size = 251.25 MiB +llama_memory_recurrent: size = 251.25 MiB ( 4 cells, 40 layers, 4 seqs), R (f32): 11.25 MiB, S (f32): 240.00 MiB +sched_reserve: reserving ... +sched_reserve: resolving fused Gated Delta Net support: +sched_reserve: fused Gated Delta Net (autoregressive) enabled +sched_reserve: fused Gated Delta Net (chunked) enabled +sched_reserve: MTL0 compute buffer size = 493.00 MiB +sched_reserve: CPU compute buffer size = 9.02 MiB +sched_reserve: graph nodes = 3749 +sched_reserve: graph splits = 2 +sched_reserve: reserve took 21.43 ms, sched copies = 1 +common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) + +system_info: n_threads = 6 (n_threads_batch = 6) / 18 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | REPACK = 1 | +kl_divergence: computing over 8 chunks, n_ctx=512, batch_size=2048, n_seq=4 +kl_divergence: 0.90 seconds per pass - ETA 0.02 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 4.2378 ± 0.5914 0.00692 ± 0.00306 0.00103 ± 0.00015 1.014 ± 0.152 % 98.824 ± 0.677 % + 2 6.0377 ± 0.7022 0.00464 ± 0.00219 0.00121 ± 0.00024 1.324 ± 0.367 % 98.431 ± 0.551 % + 3 5.0850 ± 0.4606 -0.00047 ± 0.00238 0.00202 ± 0.00024 1.601 ± 0.235 % 98.301 ± 0.468 % + 4 5.0680 ± 0.3763 -0.00097 ± 0.00195 0.00182 ± 0.00020 1.482 ± 0.193 % 98.431 ± 0.389 % + 5 5.2338 ± 0.3417 -0.00107 ± 0.00167 0.00170 ± 0.00016 1.362 ± 0.168 % 98.353 ± 0.357 % + 6 5.4076 ± 0.3215 -0.00179 ± 0.00150 0.00164 ± 0.00014 1.292 ± 0.148 % 98.431 ± 0.318 % + 7 5.5998 ± 0.3089 -0.00073 ± 0.00171 0.00159 ± 0.00012 1.252 ± 0.132 % 98.431 ± 0.294 % + 8 6.1109 ± 0.3255 -0.00078 ± 0.00160 0.00155 ± 0.00011 1.231 ± 0.119 % 98.431 ± 0.275 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 6.110915 ± 0.325525 +Mean PPL(base) : 6.115704 ± 0.325284 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.96% +Mean ln(PPL(Q)/PPL(base)) : -0.000783 ± 0.001599 +Mean PPL(Q)/PPL(base) : 0.999217 ± 0.001598 +Mean PPL(Q)-PPL(base) : -0.004789 ± 0.009765 + +====== KL divergence statistics ====== +Mean KLD: 0.001549 ± 0.000108 +Maximum KLD: 0.111541 +99.9% KLD: 0.060302 +99.0% KLD: 0.019415 +95.0% KLD: 0.005553 +90.0% KLD: 0.003112 +Median KLD: 0.000425 +10.0% KLD: 0.000026 + 5.0% KLD: 0.000007 + 1.0% KLD: -0.000000 + 0.1% KLD: -0.000008 +Minimum KLD: -0.000016 + +====== Token probability statistics ====== +Mean Δp: 0.032 ± 0.027 % +Maximum Δp: 14.606% +99.9% Δp: 10.198% +99.0% Δp: 3.727% +95.0% Δp: 1.448% +90.0% Δp: 0.746% +75.0% Δp: 0.179% +Median Δp: -0.001% +25.0% Δp: -0.164% +10.0% Δp: -0.646% + 5.0% Δp: -1.216% + 1.0% Δp: -3.295% + 0.1% Δp: -7.738% +Minimum Δp: -22.079% +RMS Δp : 1.231 ± 0.119 % +Same top p: 98.431 ± 0.275 % + +llama_perf_context_print: load time = 2303.16 ms +llama_perf_context_print: prompt eval time = 837.87 ms / 4096 tokens ( 0.20 ms per token, 4888.57 tokens per second) +llama_perf_context_print: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) +llama_perf_context_print: total time = 2089.41 ms / 4097 tokens +llama_perf_context_print: graphs reused = 2 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - MTL0 (Apple M5 Max) | 110100 = 74151 + (35948 = 35183 + 272 + 493) + 0 | +llama_memory_breakdown_print: | - Host | 524 = 515 + 0 + 9 | +ggml_metal_free: deallocating diff --git a/benchmark-results-raw/kld_moe_turbo3.log b/benchmark-results-raw/kld_moe_turbo3.log new file mode 100644 index 0000000..ce6b7c6 --- /dev/null +++ b/benchmark-results-raw/kld_moe_turbo3.log @@ -0,0 +1,274 @@ +ggml_metal_device_init: testing tensor API for f16 support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1058cecf0 | th_max = 1024 | th_width = 32 +ggml_metal_device_init: testing tensor API for bfloat support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1058ce970 | th_max = 1024 | th_width = 32 +ggml_metal_library_init: using embedded metal library +ggml_metal_library_init: turbo3 sparse V dequant enabled +ggml_metal_library_init: loaded in 0.008 sec +ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s) +ggml_metal_device_init: GPU name: MTL0 +ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010) +ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003) +ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002) +ggml_metal_device_init: simdgroup reduction = true +ggml_metal_device_init: simdgroup matrix mul. = true +ggml_metal_device_init: has unified memory = true +ggml_metal_device_init: has bfloat = true +ggml_metal_device_init: has tensor = true +ggml_metal_device_init: use residency sets = true +ggml_metal_device_init: use shared buffers = true +ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB +build: 159 (7d1bd95) with AppleClang 17.0.0.17000604 for Darwin arm64 +common_init_result: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on +llama_params_fit_impl: projected to use 35420 MiB of device memory vs. 109848 MiB of free device memory +llama_params_fit_impl: will leave 74427 >= 1024 MiB of free device memory, no changes needed +llama_params_fit: successfully fit params to free device memory +llama_params_fit: fitting params to free memory took 0.40 seconds +llama_model_load_from_file_impl: using device MTL0 (Apple M5 Max) (unknown id) - 110100 MiB free +llama_model_loader: loaded meta data with 52 key-value pairs and 733 tensors from /Users/tom/local_llms/models/Qwen3.5-35B-A3B-Q8_0.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35moe +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +llama_model_loader: - kv 5: general.name str = Qwen3.5-35B-A3B +llama_model_loader: - kv 6: general.basename str = Qwen3.5-35B-A3B +llama_model_loader: - kv 7: general.quantized_by str = Unsloth +llama_model_loader: - kv 8: general.size_label str = 35B-A3B +llama_model_loader: - kv 9: general.license str = apache-2.0 +llama_model_loader: - kv 10: general.license.link str = https://huggingface.co/Qwen/Qwen3.5-3... +llama_model_loader: - kv 11: general.repo_url str = https://huggingface.co/unsloth +llama_model_loader: - kv 12: general.base_model.count u32 = 1 +llama_model_loader: - kv 13: general.base_model.0.name str = Qwen3.5 35B A3B +llama_model_loader: - kv 14: general.base_model.0.organization str = Qwen +llama_model_loader: - kv 15: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen3.5-3... +llama_model_loader: - kv 16: general.tags arr[str,2] = ["unsloth", "image-text-to-text"] +llama_model_loader: - kv 17: qwen35moe.block_count u32 = 40 +llama_model_loader: - kv 18: qwen35moe.context_length u32 = 262144 +llama_model_loader: - kv 19: qwen35moe.embedding_length u32 = 2048 +llama_model_loader: - kv 20: qwen35moe.attention.head_count u32 = 16 +llama_model_loader: - kv 21: qwen35moe.attention.head_count_kv u32 = 2 +llama_model_loader: - kv 22: qwen35moe.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 23: qwen35moe.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 24: qwen35moe.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 25: qwen35moe.expert_count u32 = 256 +llama_model_loader: - kv 26: qwen35moe.expert_used_count u32 = 8 +llama_model_loader: - kv 27: qwen35moe.attention.key_length u32 = 256 +llama_model_loader: - kv 28: qwen35moe.attention.value_length u32 = 256 +llama_model_loader: - kv 29: qwen35moe.expert_feed_forward_length u32 = 512 +llama_model_loader: - kv 30: qwen35moe.expert_shared_feed_forward_length u32 = 512 +llama_model_loader: - kv 31: qwen35moe.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 32: qwen35moe.ssm.state_size u32 = 128 +llama_model_loader: - kv 33: qwen35moe.ssm.group_count u32 = 16 +llama_model_loader: - kv 34: qwen35moe.ssm.time_step_rank u32 = 32 +llama_model_loader: - kv 35: qwen35moe.ssm.inner_size u32 = 4096 +llama_model_loader: - kv 36: qwen35moe.full_attention_interval u32 = 4 +llama_model_loader: - kv 37: qwen35moe.rope.dimension_count u32 = 64 +llama_model_loader: - kv 38: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 39: tokenizer.ggml.pre str = qwen35 +llama_model_loader: - kv 40: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 41: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 42: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 43: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 44: tokenizer.ggml.padding_token_id u32 = 248055 +llama_model_loader: - kv 45: tokenizer.chat_template str = {%- set image_count = namespace(value... +llama_model_loader: - kv 46: general.quantization_version u32 = 2 +llama_model_loader: - kv 47: general.file_type u32 = 7 +llama_model_loader: - kv 48: quantize.imatrix.file str = Qwen3.5-35B-A3B-GGUF/imatrix_unsloth.... +llama_model_loader: - kv 49: quantize.imatrix.dataset str = unsloth_calibration_Qwen3.5-35B-A3B.txt +llama_model_loader: - kv 50: quantize.imatrix.entries_count u32 = 510 +llama_model_loader: - kv 51: quantize.imatrix.chunks_count u32 = 76 +llama_model_loader: - type f32: 301 tensors +llama_model_loader: - type q8_0: 432 tensors +print_info: file format = GGUF V3 (latest) +print_info: file type = Q8_0 +print_info: file size = 34.36 GiB (8.52 BPW) +load: 0 unused tokens +load: printing all EOG tokens: +load: - 248044 ('<|endoftext|>') +load: - 248046 ('<|im_end|>') +load: - 248063 ('<|fim_pad|>') +load: - 248064 ('<|repo_name|>') +load: - 248065 ('<|file_sep|>') +load: special tokens cache size = 33 +load: token to piece cache size = 1.7581 MB +print_info: arch = qwen35moe +print_info: vocab_only = 0 +print_info: no_alloc = 0 +print_info: n_ctx_train = 262144 +print_info: n_embd = 2048 +print_info: n_embd_inp = 2048 +print_info: n_layer = 40 +print_info: n_head = 16 +print_info: n_head_kv = 2 +print_info: n_rot = 64 +print_info: n_swa = 0 +print_info: is_swa_any = 0 +print_info: n_embd_head_k = 256 +print_info: n_embd_head_v = 256 +print_info: n_gqa = 8 +print_info: n_embd_k_gqa = 512 +print_info: n_embd_v_gqa = 512 +print_info: f_norm_eps = 0.0e+00 +print_info: f_norm_rms_eps = 1.0e-06 +print_info: f_clamp_kqv = 0.0e+00 +print_info: f_max_alibi_bias = 0.0e+00 +print_info: f_logit_scale = 0.0e+00 +print_info: f_attn_scale = 0.0e+00 +print_info: n_ff = 0 +print_info: n_expert = 256 +print_info: n_expert_used = 8 +print_info: n_expert_groups = 0 +print_info: n_group_used = 0 +print_info: causal attn = 1 +print_info: pooling type = -1 +print_info: rope type = 40 +print_info: rope scaling = linear +print_info: freq_base_train = 10000000.0 +print_info: freq_scale_train = 1 +print_info: n_ctx_orig_yarn = 262144 +print_info: rope_yarn_log_mul = 0.0000 +print_info: rope_finetuned = unknown +print_info: mrope sections = [11, 11, 10, 0] +print_info: ssm_d_conv = 4 +print_info: ssm_d_inner = 4096 +print_info: ssm_d_state = 128 +print_info: ssm_dt_rank = 32 +print_info: ssm_n_group = 16 +print_info: ssm_dt_b_c_rms = 0 +print_info: model type = 35B.A3B +print_info: model params = 34.66 B +print_info: general.name = Qwen3.5-35B-A3B +print_info: vocab type = BPE +print_info: n_vocab = 248320 +print_info: n_merges = 247587 +print_info: BOS token = 11 ',' +print_info: EOS token = 248046 '<|im_end|>' +print_info: EOT token = 248046 '<|im_end|>' +print_info: PAD token = 248055 '<|vision_pad|>' +print_info: LF token = 198 'Ċ' +print_info: FIM PRE token = 248060 '<|fim_prefix|>' +print_info: FIM SUF token = 248062 '<|fim_suffix|>' +print_info: FIM MID token = 248061 '<|fim_middle|>' +print_info: FIM PAD token = 248063 '<|fim_pad|>' +print_info: FIM REP token = 248064 '<|repo_name|>' +print_info: FIM SEP token = 248065 '<|file_sep|>' +print_info: EOG token = 248044 '<|endoftext|>' +print_info: EOG token = 248046 '<|im_end|>' +print_info: EOG token = 248063 '<|fim_pad|>' +print_info: EOG token = 248064 '<|repo_name|>' +print_info: EOG token = 248065 '<|file_sep|>' +print_info: max token length = 256 +load_tensors: loading model tensors, this can take a while... (mmap = true, direct_io = false) +load_tensors: offloading output layer to GPU +load_tensors: offloading 39 repeating layers to GPU +load_tensors: offloaded 41/41 layers to GPU +load_tensors: CPU_Mapped model buffer size = 515.31 MiB +load_tensors: MTL0_Mapped model buffer size = 35183.10 MiB +.................................................................................................... +common_init_result: added <|endoftext|> logit bias = -inf +common_init_result: added <|im_end|> logit bias = -inf +common_init_result: added <|fim_pad|> logit bias = -inf +common_init_result: added <|repo_name|> logit bias = -inf +common_init_result: added <|file_sep|> logit bias = -inf +llama_context: constructing llama_context +llama_context: n_seq_max = 4 +llama_context: n_ctx = 2048 +llama_context: n_ctx_seq = 512 +llama_context: n_batch = 2048 +llama_context: n_ubatch = 512 +llama_context: causal_attn = 1 +llama_context: flash_attn = enabled +llama_context: kv_unified = false +llama_context: freq_base = 10000000.0 +llama_context: freq_scale = 1 +llama_context: n_ctx_seq (512) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +ggml_metal_init: allocating +ggml_metal_init: found device: Apple M5 Max +ggml_metal_init: picking default device: Apple M5 Max +ggml_metal_init: use fusion = true +ggml_metal_init: use concurrency = true +ggml_metal_init: use graph optimize = true +llama_context: CPU output buffer size = 3.79 MiB +llama_kv_cache: MTL0 KV buffer size = 8.88 MiB +llama_kv_cache: TurboQuant rotation matrices initialized (128x128) +llama_kv_cache: size = 8.75 MiB ( 512 cells, 10 layers, 4/4 seqs), K (turbo3): 4.38 MiB, V (turbo3): 4.38 MiB +llama_memory_recurrent: MTL0 RS buffer size = 251.25 MiB +llama_memory_recurrent: size = 251.25 MiB ( 4 cells, 40 layers, 4 seqs), R (f32): 11.25 MiB, S (f32): 240.00 MiB +sched_reserve: reserving ... +sched_reserve: resolving fused Gated Delta Net support: +sched_reserve: fused Gated Delta Net (autoregressive) enabled +sched_reserve: fused Gated Delta Net (chunked) enabled +sched_reserve: MTL0 compute buffer size = 493.00 MiB +sched_reserve: CPU compute buffer size = 9.02 MiB +sched_reserve: graph nodes = 3769 +sched_reserve: graph splits = 2 +sched_reserve: reserve took 21.93 ms, sched copies = 1 +common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) + +system_info: n_threads = 6 (n_threads_batch = 6) / 18 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | REPACK = 1 | +kl_divergence: computing over 8 chunks, n_ctx=512, batch_size=2048, n_seq=4 +kl_divergence: 0.91 seconds per pass - ETA 0.02 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 4.2747 ± 0.5995 0.01561 ± 0.00921 0.01045 ± 0.00093 3.264 ± 0.338 % 97.647 ± 0.951 % + 2 6.1278 ± 0.7201 0.01944 ± 0.00713 0.01287 ± 0.00078 3.241 ± 0.254 % 95.294 ± 0.939 % + 3 5.1620 ± 0.4723 0.01454 ± 0.00683 0.01830 ± 0.00184 4.907 ± 0.519 % 94.379 ± 0.833 % + 4 5.1304 ± 0.3841 0.01125 ± 0.00564 0.01731 ± 0.00146 4.605 ± 0.433 % 94.510 ± 0.714 % + 5 5.3003 ± 0.3479 0.01156 ± 0.00510 0.01676 ± 0.00121 4.468 ± 0.374 % 94.745 ± 0.625 % + 6 5.4608 ± 0.3261 0.00801 ± 0.00463 0.01644 ± 0.00103 4.343 ± 0.324 % 94.706 ± 0.573 % + 7 5.6585 ± 0.3139 0.00969 ± 0.00451 0.01627 ± 0.00089 4.187 ± 0.289 % 94.454 ± 0.542 % + 8 6.1756 ± 0.3303 0.00974 ± 0.00415 0.01615 ± 0.00080 4.090 ± 0.263 % 94.314 ± 0.513 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 6.175580 ± 0.330342 +Mean PPL(base) : 6.115704 ± 0.325284 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.70% +Mean ln(PPL(Q)/PPL(base)) : 0.009743 ± 0.004153 +Mean PPL(Q)/PPL(base) : 1.009790 ± 0.004193 +Mean PPL(Q)-PPL(base) : 0.059876 ± 0.025950 + +====== KL divergence statistics ====== +Mean KLD: 0.016145 ± 0.000801 +Maximum KLD: 1.165365 +99.9% KLD: 0.338142 +99.0% KLD: 0.109562 +95.0% KLD: 0.052614 +90.0% KLD: 0.035231 +Median KLD: 0.008930 +10.0% KLD: 0.000614 + 5.0% KLD: 0.000174 + 1.0% KLD: 0.000019 + 0.1% KLD: -0.000000 +Minimum KLD: -0.000001 + +====== Token probability statistics ====== +Mean Δp: -0.252 ± 0.090 % +Maximum Δp: 54.667% +99.9% Δp: 29.065% +99.0% Δp: 10.393% +95.0% Δp: 4.918% +90.0% Δp: 2.814% +75.0% Δp: 0.584% +Median Δp: -0.025% +25.0% Δp: -1.042% +10.0% Δp: -3.569% + 5.0% Δp: -5.758% + 1.0% Δp: -12.187% + 0.1% Δp: -33.633% +Minimum Δp: -34.136% +RMS Δp : 4.090 ± 0.263 % +Same top p: 94.314 ± 0.513 % + +llama_perf_context_print: load time = 2179.58 ms +llama_perf_context_print: prompt eval time = 855.23 ms / 4096 tokens ( 0.21 ms per token, 4789.34 tokens per second) +llama_perf_context_print: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) +llama_perf_context_print: total time = 2122.67 ms / 4097 tokens +llama_perf_context_print: graphs reused = 2 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - MTL0 (Apple M5 Max) | 110100 = 74163 + (35936 = 35183 + 260 + 493) + 0 | +llama_memory_breakdown_print: | - Host | 524 = 515 + 0 + 9 | +ggml_metal_free: deallocating diff --git a/benchmark-results-raw/ppl_f16.log b/benchmark-results-raw/ppl_f16.log new file mode 100644 index 0000000..232df6f --- /dev/null +++ b/benchmark-results-raw/ppl_f16.log @@ -0,0 +1,226 @@ +ggml_metal_device_init: testing tensor API for f16 support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x104f498f0 | th_max = 1024 | th_width = 32 +ggml_metal_device_init: testing tensor API for bfloat support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x104f4a070 | th_max = 1024 | th_width = 32 +ggml_metal_library_init: using embedded metal library +ggml_metal_library_init: loaded in 0.014 sec +ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s) +ggml_metal_device_init: GPU name: MTL0 +ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010) +ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003) +ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002) +ggml_metal_device_init: simdgroup reduction = true +ggml_metal_device_init: simdgroup matrix mul. = true +ggml_metal_device_init: has unified memory = true +ggml_metal_device_init: has bfloat = true +ggml_metal_device_init: has tensor = true +ggml_metal_device_init: use residency sets = true +ggml_metal_device_init: use shared buffers = true +ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB +build: 23 (a696962) with AppleClang 17.0.0.17000604 for Darwin arm64 +common_init_result: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on +llama_params_fit_impl: projected to use 35452 MiB of device memory vs. 109848 MiB of free device memory +llama_params_fit_impl: will leave 74396 >= 1024 MiB of free device memory, no changes needed +llama_params_fit: successfully fit params to free device memory +llama_params_fit: fitting params to free memory took 0.25 seconds +llama_model_load_from_file_impl: using device MTL0 (Apple M5 Max) (unknown id) - 110100 MiB free +llama_model_loader: loaded meta data with 52 key-value pairs and 733 tensors from /Users/tom/local_llms/models/Qwen3.5-35B-A3B-Q8_0.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35moe +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +llama_model_loader: - kv 5: general.name str = Qwen3.5-35B-A3B +llama_model_loader: - kv 6: general.basename str = Qwen3.5-35B-A3B +llama_model_loader: - kv 7: general.quantized_by str = Unsloth +llama_model_loader: - kv 8: general.size_label str = 35B-A3B +llama_model_loader: - kv 9: general.license str = apache-2.0 +llama_model_loader: - kv 10: general.license.link str = https://huggingface.co/Qwen/Qwen3.5-3... +llama_model_loader: - kv 11: general.repo_url str = https://huggingface.co/unsloth +llama_model_loader: - kv 12: general.base_model.count u32 = 1 +llama_model_loader: - kv 13: general.base_model.0.name str = Qwen3.5 35B A3B +llama_model_loader: - kv 14: general.base_model.0.organization str = Qwen +llama_model_loader: - kv 15: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen3.5-3... +llama_model_loader: - kv 16: general.tags arr[str,2] = ["unsloth", "image-text-to-text"] +llama_model_loader: - kv 17: qwen35moe.block_count u32 = 40 +llama_model_loader: - kv 18: qwen35moe.context_length u32 = 262144 +llama_model_loader: - kv 19: qwen35moe.embedding_length u32 = 2048 +llama_model_loader: - kv 20: qwen35moe.attention.head_count u32 = 16 +llama_model_loader: - kv 21: qwen35moe.attention.head_count_kv u32 = 2 +llama_model_loader: - kv 22: qwen35moe.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 23: qwen35moe.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 24: qwen35moe.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 25: qwen35moe.expert_count u32 = 256 +llama_model_loader: - kv 26: qwen35moe.expert_used_count u32 = 8 +llama_model_loader: - kv 27: qwen35moe.attention.key_length u32 = 256 +llama_model_loader: - kv 28: qwen35moe.attention.value_length u32 = 256 +llama_model_loader: - kv 29: qwen35moe.expert_feed_forward_length u32 = 512 +llama_model_loader: - kv 30: qwen35moe.expert_shared_feed_forward_length u32 = 512 +llama_model_loader: - kv 31: qwen35moe.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 32: qwen35moe.ssm.state_size u32 = 128 +llama_model_loader: - kv 33: qwen35moe.ssm.group_count u32 = 16 +llama_model_loader: - kv 34: qwen35moe.ssm.time_step_rank u32 = 32 +llama_model_loader: - kv 35: qwen35moe.ssm.inner_size u32 = 4096 +llama_model_loader: - kv 36: qwen35moe.full_attention_interval u32 = 4 +llama_model_loader: - kv 37: qwen35moe.rope.dimension_count u32 = 64 +llama_model_loader: - kv 38: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 39: tokenizer.ggml.pre str = qwen35 +llama_model_loader: - kv 40: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 41: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 42: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 43: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 44: tokenizer.ggml.padding_token_id u32 = 248055 +llama_model_loader: - kv 45: tokenizer.chat_template str = {%- set image_count = namespace(value... +llama_model_loader: - kv 46: general.quantization_version u32 = 2 +llama_model_loader: - kv 47: general.file_type u32 = 7 +llama_model_loader: - kv 48: quantize.imatrix.file str = Qwen3.5-35B-A3B-GGUF/imatrix_unsloth.... +llama_model_loader: - kv 49: quantize.imatrix.dataset str = unsloth_calibration_Qwen3.5-35B-A3B.txt +llama_model_loader: - kv 50: quantize.imatrix.entries_count u32 = 510 +llama_model_loader: - kv 51: quantize.imatrix.chunks_count u32 = 76 +llama_model_loader: - type f32: 301 tensors +llama_model_loader: - type q8_0: 432 tensors +print_info: file format = GGUF V3 (latest) +print_info: file type = Q8_0 +print_info: file size = 34.36 GiB (8.52 BPW) +load: 0 unused tokens +load: printing all EOG tokens: +load: - 248044 ('<|endoftext|>') +load: - 248046 ('<|im_end|>') +load: - 248063 ('<|fim_pad|>') +load: - 248064 ('<|repo_name|>') +load: - 248065 ('<|file_sep|>') +load: special tokens cache size = 33 +load: token to piece cache size = 1.7581 MB +print_info: arch = qwen35moe +print_info: vocab_only = 0 +print_info: no_alloc = 0 +print_info: n_ctx_train = 262144 +print_info: n_embd = 2048 +print_info: n_embd_inp = 2048 +print_info: n_layer = 40 +print_info: n_head = 16 +print_info: n_head_kv = 2 +print_info: n_rot = 64 +print_info: n_swa = 0 +print_info: is_swa_any = 0 +print_info: n_embd_head_k = 256 +print_info: n_embd_head_v = 256 +print_info: n_gqa = 8 +print_info: n_embd_k_gqa = 512 +print_info: n_embd_v_gqa = 512 +print_info: f_norm_eps = 0.0e+00 +print_info: f_norm_rms_eps = 1.0e-06 +print_info: f_clamp_kqv = 0.0e+00 +print_info: f_max_alibi_bias = 0.0e+00 +print_info: f_logit_scale = 0.0e+00 +print_info: f_attn_scale = 0.0e+00 +print_info: n_ff = 0 +print_info: n_expert = 256 +print_info: n_expert_used = 8 +print_info: n_expert_groups = 0 +print_info: n_group_used = 0 +print_info: causal attn = 1 +print_info: pooling type = 0 +print_info: rope type = 40 +print_info: rope scaling = linear +print_info: freq_base_train = 10000000.0 +print_info: freq_scale_train = 1 +print_info: n_ctx_orig_yarn = 262144 +print_info: rope_yarn_log_mul = 0.0000 +print_info: rope_finetuned = unknown +print_info: mrope sections = [11, 11, 10, 0] +print_info: ssm_d_conv = 4 +print_info: ssm_d_inner = 4096 +print_info: ssm_d_state = 128 +print_info: ssm_dt_rank = 32 +print_info: ssm_n_group = 16 +print_info: ssm_dt_b_c_rms = 0 +print_info: model type = 35B.A3B +print_info: model params = 34.66 B +print_info: general.name = Qwen3.5-35B-A3B +print_info: vocab type = BPE +print_info: n_vocab = 248320 +print_info: n_merges = 247587 +print_info: BOS token = 11 ',' +print_info: EOS token = 248046 '<|im_end|>' +print_info: EOT token = 248046 '<|im_end|>' +print_info: PAD token = 248055 '<|vision_pad|>' +print_info: LF token = 198 'Ċ' +print_info: FIM PRE token = 248060 '<|fim_prefix|>' +print_info: FIM SUF token = 248062 '<|fim_suffix|>' +print_info: FIM MID token = 248061 '<|fim_middle|>' +print_info: FIM PAD token = 248063 '<|fim_pad|>' +print_info: FIM REP token = 248064 '<|repo_name|>' +print_info: FIM SEP token = 248065 '<|file_sep|>' +print_info: EOG token = 248044 '<|endoftext|>' +print_info: EOG token = 248046 '<|im_end|>' +print_info: EOG token = 248063 '<|fim_pad|>' +print_info: EOG token = 248064 '<|repo_name|>' +print_info: EOG token = 248065 '<|file_sep|>' +print_info: max token length = 256 +load_tensors: loading model tensors, this can take a while... (mmap = true, direct_io = false) +load_tensors: offloading output layer to GPU +load_tensors: offloading 39 repeating layers to GPU +load_tensors: offloaded 41/41 layers to GPU +load_tensors: CPU_Mapped model buffer size = 515.31 MiB +load_tensors: MTL0_Mapped model buffer size = 35183.10 MiB +.................................................................................................... +common_init_result: added <|endoftext|> logit bias = -inf +common_init_result: added <|im_end|> logit bias = -inf +common_init_result: added <|fim_pad|> logit bias = -inf +common_init_result: added <|repo_name|> logit bias = -inf +common_init_result: added <|file_sep|> logit bias = -inf +llama_context: constructing llama_context +llama_context: n_seq_max = 4 +llama_context: n_ctx = 2048 +llama_context: n_ctx_seq = 512 +llama_context: n_batch = 2048 +llama_context: n_ubatch = 512 +llama_context: causal_attn = 1 +llama_context: flash_attn = enabled +llama_context: kv_unified = false +llama_context: freq_base = 10000000.0 +llama_context: freq_scale = 1 +llama_context: n_ctx_seq (512) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +ggml_metal_init: allocating +ggml_metal_init: found device: Apple M5 Max +ggml_metal_init: picking default device: Apple M5 Max +ggml_metal_init: use fusion = true +ggml_metal_init: use concurrency = true +ggml_metal_init: use graph optimize = true +llama_context: CPU output buffer size = 3.79 MiB +llama_kv_cache: MTL0 KV buffer size = 40.00 MiB +llama_kv_cache: size = 40.00 MiB ( 512 cells, 10 layers, 4/4 seqs), K (f16): 20.00 MiB, V (f16): 20.00 MiB +llama_memory_recurrent: MTL0 RS buffer size = 251.25 MiB +llama_memory_recurrent: size = 251.25 MiB ( 4 cells, 40 layers, 4 seqs), R (f32): 11.25 MiB, S (f32): 240.00 MiB +sched_reserve: reserving ... +sched_reserve: resolving fused Gated Delta Net support: +sched_reserve: fused Gated Delta Net (autoregressive) enabled +sched_reserve: fused Gated Delta Net (chunked) enabled +sched_reserve: MTL0 compute buffer size = 493.00 MiB +sched_reserve: CPU compute buffer size = 9.02 MiB +sched_reserve: graph nodes = 3749 +sched_reserve: graph splits = 2 +sched_reserve: reserve took 69.98 ms, sched copies = 1 +common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) + +system_info: n_threads = 6 (n_threads_batch = 6) / 18 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | REPACK = 1 | +perplexity: tokenizing the input .. +perplexity: tokenization took 413.009 ms +perplexity: calculating perplexity over 8 chunks, n_ctx=512, batch_size=2048, n_seq=4 +perplexity: 1.13 seconds per pass - ETA 0.03 minutes +[1]4.2085,[2]6.0098,[3]5.0874,[4]5.0730,[5]5.2394,[6]5.4173,[7]5.6095,[8]6.1210, +Final estimate: PPL = 6.1210 +/- 0.32631 + +llama_perf_context_print: load time = 10433.17 ms +llama_perf_context_print: prompt eval time = 835.30 ms / 4096 tokens ( 0.20 ms per token, 4903.60 tokens per second) +llama_perf_context_print: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) +llama_perf_context_print: total time = 2570.62 ms / 4097 tokens +llama_perf_context_print: graphs reused = 2 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - MTL0 (Apple M5 Max) | 110100 = 74132 + (35967 = 35183 + 291 + 493) + 0 | +llama_memory_breakdown_print: | - Host | 524 = 515 + 0 + 9 | +ggml_metal_free: deallocating diff --git a/benchmark-results-raw/ppl_turbo3.log b/benchmark-results-raw/ppl_turbo3.log new file mode 100644 index 0000000..9ca5469 --- /dev/null +++ b/benchmark-results-raw/ppl_turbo3.log @@ -0,0 +1,227 @@ +ggml_metal_device_init: testing tensor API for f16 support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1011e1410 | th_max = 1024 | th_width = 32 +ggml_metal_device_init: testing tensor API for bfloat support +ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel' +ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1011e2a10 | th_max = 1024 | th_width = 32 +ggml_metal_library_init: using embedded metal library +ggml_metal_library_init: loaded in 6.225 sec +ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s) +ggml_metal_device_init: GPU name: MTL0 +ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010) +ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003) +ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002) +ggml_metal_device_init: simdgroup reduction = true +ggml_metal_device_init: simdgroup matrix mul. = true +ggml_metal_device_init: has unified memory = true +ggml_metal_device_init: has bfloat = true +ggml_metal_device_init: has tensor = true +ggml_metal_device_init: use residency sets = true +ggml_metal_device_init: use shared buffers = true +ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB +build: 25 (76c5024) with AppleClang 17.0.0.17000604 for Darwin arm64 +common_init_result: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on +llama_params_fit_impl: projected to use 35420 MiB of device memory vs. 109848 MiB of free device memory +llama_params_fit_impl: will leave 74427 >= 1024 MiB of free device memory, no changes needed +llama_params_fit: successfully fit params to free device memory +llama_params_fit: fitting params to free memory took 0.22 seconds +llama_model_load_from_file_impl: using device MTL0 (Apple M5 Max) (unknown id) - 110100 MiB free +llama_model_loader: loaded meta data with 52 key-value pairs and 733 tensors from /Users/tom/local_llms/models/Qwen3.5-35B-A3B-Q8_0.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35moe +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +llama_model_loader: - kv 5: general.name str = Qwen3.5-35B-A3B +llama_model_loader: - kv 6: general.basename str = Qwen3.5-35B-A3B +llama_model_loader: - kv 7: general.quantized_by str = Unsloth +llama_model_loader: - kv 8: general.size_label str = 35B-A3B +llama_model_loader: - kv 9: general.license str = apache-2.0 +llama_model_loader: - kv 10: general.license.link str = https://huggingface.co/Qwen/Qwen3.5-3... +llama_model_loader: - kv 11: general.repo_url str = https://huggingface.co/unsloth +llama_model_loader: - kv 12: general.base_model.count u32 = 1 +llama_model_loader: - kv 13: general.base_model.0.name str = Qwen3.5 35B A3B +llama_model_loader: - kv 14: general.base_model.0.organization str = Qwen +llama_model_loader: - kv 15: general.base_model.0.repo_url str = https://huggingface.co/Qwen/Qwen3.5-3... +llama_model_loader: - kv 16: general.tags arr[str,2] = ["unsloth", "image-text-to-text"] +llama_model_loader: - kv 17: qwen35moe.block_count u32 = 40 +llama_model_loader: - kv 18: qwen35moe.context_length u32 = 262144 +llama_model_loader: - kv 19: qwen35moe.embedding_length u32 = 2048 +llama_model_loader: - kv 20: qwen35moe.attention.head_count u32 = 16 +llama_model_loader: - kv 21: qwen35moe.attention.head_count_kv u32 = 2 +llama_model_loader: - kv 22: qwen35moe.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 23: qwen35moe.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 24: qwen35moe.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 25: qwen35moe.expert_count u32 = 256 +llama_model_loader: - kv 26: qwen35moe.expert_used_count u32 = 8 +llama_model_loader: - kv 27: qwen35moe.attention.key_length u32 = 256 +llama_model_loader: - kv 28: qwen35moe.attention.value_length u32 = 256 +llama_model_loader: - kv 29: qwen35moe.expert_feed_forward_length u32 = 512 +llama_model_loader: - kv 30: qwen35moe.expert_shared_feed_forward_length u32 = 512 +llama_model_loader: - kv 31: qwen35moe.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 32: qwen35moe.ssm.state_size u32 = 128 +llama_model_loader: - kv 33: qwen35moe.ssm.group_count u32 = 16 +llama_model_loader: - kv 34: qwen35moe.ssm.time_step_rank u32 = 32 +llama_model_loader: - kv 35: qwen35moe.ssm.inner_size u32 = 4096 +llama_model_loader: - kv 36: qwen35moe.full_attention_interval u32 = 4 +llama_model_loader: - kv 37: qwen35moe.rope.dimension_count u32 = 64 +llama_model_loader: - kv 38: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 39: tokenizer.ggml.pre str = qwen35 +llama_model_loader: - kv 40: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 41: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 42: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 43: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 44: tokenizer.ggml.padding_token_id u32 = 248055 +llama_model_loader: - kv 45: tokenizer.chat_template str = {%- set image_count = namespace(value... +llama_model_loader: - kv 46: general.quantization_version u32 = 2 +llama_model_loader: - kv 47: general.file_type u32 = 7 +llama_model_loader: - kv 48: quantize.imatrix.file str = Qwen3.5-35B-A3B-GGUF/imatrix_unsloth.... +llama_model_loader: - kv 49: quantize.imatrix.dataset str = unsloth_calibration_Qwen3.5-35B-A3B.txt +llama_model_loader: - kv 50: quantize.imatrix.entries_count u32 = 510 +llama_model_loader: - kv 51: quantize.imatrix.chunks_count u32 = 76 +llama_model_loader: - type f32: 301 tensors +llama_model_loader: - type q8_0: 432 tensors +print_info: file format = GGUF V3 (latest) +print_info: file type = Q8_0 +print_info: file size = 34.36 GiB (8.52 BPW) +load: 0 unused tokens +load: printing all EOG tokens: +load: - 248044 ('<|endoftext|>') +load: - 248046 ('<|im_end|>') +load: - 248063 ('<|fim_pad|>') +load: - 248064 ('<|repo_name|>') +load: - 248065 ('<|file_sep|>') +load: special tokens cache size = 33 +load: token to piece cache size = 1.7581 MB +print_info: arch = qwen35moe +print_info: vocab_only = 0 +print_info: no_alloc = 0 +print_info: n_ctx_train = 262144 +print_info: n_embd = 2048 +print_info: n_embd_inp = 2048 +print_info: n_layer = 40 +print_info: n_head = 16 +print_info: n_head_kv = 2 +print_info: n_rot = 64 +print_info: n_swa = 0 +print_info: is_swa_any = 0 +print_info: n_embd_head_k = 256 +print_info: n_embd_head_v = 256 +print_info: n_gqa = 8 +print_info: n_embd_k_gqa = 512 +print_info: n_embd_v_gqa = 512 +print_info: f_norm_eps = 0.0e+00 +print_info: f_norm_rms_eps = 1.0e-06 +print_info: f_clamp_kqv = 0.0e+00 +print_info: f_max_alibi_bias = 0.0e+00 +print_info: f_logit_scale = 0.0e+00 +print_info: f_attn_scale = 0.0e+00 +print_info: n_ff = 0 +print_info: n_expert = 256 +print_info: n_expert_used = 8 +print_info: n_expert_groups = 0 +print_info: n_group_used = 0 +print_info: causal attn = 1 +print_info: pooling type = 0 +print_info: rope type = 40 +print_info: rope scaling = linear +print_info: freq_base_train = 10000000.0 +print_info: freq_scale_train = 1 +print_info: n_ctx_orig_yarn = 262144 +print_info: rope_yarn_log_mul = 0.0000 +print_info: rope_finetuned = unknown +print_info: mrope sections = [11, 11, 10, 0] +print_info: ssm_d_conv = 4 +print_info: ssm_d_inner = 4096 +print_info: ssm_d_state = 128 +print_info: ssm_dt_rank = 32 +print_info: ssm_n_group = 16 +print_info: ssm_dt_b_c_rms = 0 +print_info: model type = 35B.A3B +print_info: model params = 34.66 B +print_info: general.name = Qwen3.5-35B-A3B +print_info: vocab type = BPE +print_info: n_vocab = 248320 +print_info: n_merges = 247587 +print_info: BOS token = 11 ',' +print_info: EOS token = 248046 '<|im_end|>' +print_info: EOT token = 248046 '<|im_end|>' +print_info: PAD token = 248055 '<|vision_pad|>' +print_info: LF token = 198 'Ċ' +print_info: FIM PRE token = 248060 '<|fim_prefix|>' +print_info: FIM SUF token = 248062 '<|fim_suffix|>' +print_info: FIM MID token = 248061 '<|fim_middle|>' +print_info: FIM PAD token = 248063 '<|fim_pad|>' +print_info: FIM REP token = 248064 '<|repo_name|>' +print_info: FIM SEP token = 248065 '<|file_sep|>' +print_info: EOG token = 248044 '<|endoftext|>' +print_info: EOG token = 248046 '<|im_end|>' +print_info: EOG token = 248063 '<|fim_pad|>' +print_info: EOG token = 248064 '<|repo_name|>' +print_info: EOG token = 248065 '<|file_sep|>' +print_info: max token length = 256 +load_tensors: loading model tensors, this can take a while... (mmap = true, direct_io = false) +load_tensors: offloading output layer to GPU +load_tensors: offloading 39 repeating layers to GPU +load_tensors: offloaded 41/41 layers to GPU +load_tensors: CPU_Mapped model buffer size = 515.31 MiB +load_tensors: MTL0_Mapped model buffer size = 35183.10 MiB +.................................................................................................... +common_init_result: added <|endoftext|> logit bias = -inf +common_init_result: added <|im_end|> logit bias = -inf +common_init_result: added <|fim_pad|> logit bias = -inf +common_init_result: added <|repo_name|> logit bias = -inf +common_init_result: added <|file_sep|> logit bias = -inf +llama_context: constructing llama_context +llama_context: n_seq_max = 4 +llama_context: n_ctx = 2048 +llama_context: n_ctx_seq = 512 +llama_context: n_batch = 2048 +llama_context: n_ubatch = 512 +llama_context: causal_attn = 1 +llama_context: flash_attn = enabled +llama_context: kv_unified = false +llama_context: freq_base = 10000000.0 +llama_context: freq_scale = 1 +llama_context: n_ctx_seq (512) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +ggml_metal_init: allocating +ggml_metal_init: found device: Apple M5 Max +ggml_metal_init: picking default device: Apple M5 Max +ggml_metal_init: use fusion = true +ggml_metal_init: use concurrency = true +ggml_metal_init: use graph optimize = true +llama_context: CPU output buffer size = 3.79 MiB +llama_kv_cache: MTL0 KV buffer size = 8.81 MiB +llama_kv_cache: TurboQuant rotation matrix initialized (128x128) +llama_kv_cache: size = 8.75 MiB ( 512 cells, 10 layers, 4/4 seqs), K (turbo3): 4.38 MiB, V (turbo3): 4.38 MiB +llama_memory_recurrent: MTL0 RS buffer size = 251.25 MiB +llama_memory_recurrent: size = 251.25 MiB ( 4 cells, 40 layers, 4 seqs), R (f32): 11.25 MiB, S (f32): 240.00 MiB +sched_reserve: reserving ... +sched_reserve: resolving fused Gated Delta Net support: +sched_reserve: fused Gated Delta Net (autoregressive) enabled +sched_reserve: fused Gated Delta Net (chunked) enabled +sched_reserve: MTL0 compute buffer size = 493.00 MiB +sched_reserve: CPU compute buffer size = 9.02 MiB +sched_reserve: graph nodes = 3749 +sched_reserve: graph splits = 2 +sched_reserve: reserve took 18.17 ms, sched copies = 1 +common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) + +system_info: n_threads = 6 (n_threads_batch = 6) / 18 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | SME = 1 | ACCELERATE = 1 | REPACK = 1 | +perplexity: tokenizing the input .. +perplexity: tokenization took 344.995 ms +perplexity: calculating perplexity over 8 chunks, n_ctx=512, batch_size=2048, n_seq=4 +perplexity: 0.96 seconds per pass - ETA 0.02 minutes +[1]243.8899,[2]361.9110,[3]249.6140,[4]194.3285,[5]182.2508,[6]173.3638,[7]158.2489,[8]165.6431, +Final estimate: PPL = 165.6431 +/- 11.07513 + +llama_perf_context_print: load time = 2483.93 ms +llama_perf_context_print: prompt eval time = 836.90 ms / 4096 tokens ( 0.20 ms per token, 4894.26 tokens per second) +llama_perf_context_print: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) +llama_perf_context_print: total time = 2240.93 ms / 4097 tokens +llama_perf_context_print: graphs reused = 2 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - MTL0 (Apple M5 Max) | 110100 = 74163 + (35936 = 35183 + 260 + 493) + 0 | +llama_memory_breakdown_print: | - Host | 524 = 515 + 0 + 9 | +ggml_metal_free: deallocating diff --git a/docs/papers/turbo4-resurrection.md b/docs/papers/turbo4-resurrection.md new file mode 100644 index 0000000..8819840 --- /dev/null +++ b/docs/papers/turbo4-resurrection.md @@ -0,0 +1,298 @@ +# Resurrecting TurboQuant4: From PPL 679 to Beating q4_0 + +**Tom Turney** +Independent Researcher +GitHub: [@TheTom](https://github.com/TheTom) + +--- + +## Abstract + +TurboQuant4 (turbo4) was broken. Completely broken — PPL 679 on Metal, catastrophically slow prefill on CUDA, and a 1-bit error correction mechanism (QJL) that actively made quality worse. The community consensus was "do not bother using turbo4." + +This document describes how we took turbo4 from unusable to the best quantized KV cache format we've tested: PPL +0.23% vs q8_0, better than q4_0, matching turbo3 decode speed, and near-parity prefill. The path involved finding 7 bugs, running a QJL ablation that proved the paper's own correction mechanism is harmful for attention, and redesigning the format around 4-bit PolarQuant with optimal centroids. + +--- + +## 1. Where We Started + +### The State of turbo4 on March 28, 2026 + +turbo4 was the 4-bit variant of TurboQuant: 3-bit PolarQuant quantization of WHT-rotated KV cache vectors, plus a 1-bit QJL (Quantized Johnson-Lindenstrauss) residual correction. 4.25 bits per value, 3.76x compression. + +On paper, turbo4 should have been strictly better than turbo3 (3.5 bits). The extra bit was supposed to correct the PolarQuant residual error via QJL projection, giving near-lossless quality at only marginally less compression. + +In practice: +- **Metal (our fork):** PPL 679.27. Completely unusable. The SET_ROWS kernel was writing turbo3-format data into turbo4 blocks. +- **CUDA (buun's fork):** PPL degraded from -0.28% at 2K to +3.69% at 64K. QJL noise accumulated with context length. Prefill was 51.9% of q8_0 — catastrophically slow. +- **Community verdict:** "@spiritbuun: turbo3 is better in every way. Do not bother using turbo4." + +### The Numbers That Mattered + +| Metric | turbo4 (broken) | turbo3 | q8_0 | +|--------|----------------|--------|------| +| PPL (Metal) | 679.27 | 6.1756 | 6.1109 | +| Decode (Metal) | 44.43 tok/s | 78.24 | 86.01 | +| PPL 64K (CUDA) | +3.69% | +0.49% | baseline | +| Prefill (CUDA) | 51.9% of q8_0 | 99.3% | baseline | + +--- + +## 2. Finding the Bugs + +A code analysis of the turbo4 Metal path revealed 7 bugs. Three were critical, explaining the PPL 679. + +### Bug 1: SET_ROWS Used turbo3 Packing for turbo4 Blocks + +The `kernel_set_rows_turbo` was a shared template for both turbo3 and turbo4. It used turbo3's 2+1 bit packing scheme (2 low bits in `qs[]`, 1 high bit in `signs[]`) for 32-element blocks. But turbo4 uses 128-element blocks with packed 3-bit indices spanning byte boundaries. The shared template wrote turbo3-format data into turbo4 blocks, corrupting every KV cache write. + +**Fix:** Dedicated `kernel_set_rows_turbo4` with correct 128-element block handling. + +**Impact:** PPL 679 → 6.19. + +### Bug 2: QJL Matrix Multiplication Missing in Dequant + +The Metal dequant read QJL signs as raw ±1 values and multiplied them by a scale factor. The C reference code applied a full matrix-vector multiply (`turbo_qjl_matrix_t × signs`). Without the matrix multiply, the QJL "correction" was random noise. + +### Bug 3: SET_ROWS Missing QJL Step Entirely + +The SET_ROWS kernel performed PolarQuant quantization but never computed the QJL residual, projection, or signs. The `signs[]` field in the KV cache was either zero-initialized or contained stale data. The dequant then "corrected" with this garbage. + +### Bugs 4-7: Supporting Issues + +4. **3-bit packing format mismatch** in SET_ROWS (turbo3 2+1 scheme applied to turbo4's packed 3-bit format) +5. **rnorm uninitialized** in C reference quantizer (QJL scale computed from garbage) +6. **Metal computed rnorm from wrong-basis residual** (residual in rotated space, not normalized space) +7. **WHT used instead of QJL random matrix** in Metal (may have been intentional approximation, but breaks C reference symmetry) + +--- + +## 3. The QJL Ablation + +After fixing Bug 1 (SET_ROWS), turbo4 PPL dropped from 679 to 6.19. Still slightly worse than turbo3 (6.18). The natural question: is QJL actually helping? + +We disabled QJL in the dequant by removing the sign correction: + +```metal +// Original: cache[i] = (recon[i] + signs_f[i] * qjl_scale) * norm; +cache[i] = recon[i] * norm; // PolarQuant only +``` + +**Result:** + +| Config | PPL | +|--------|-----| +| turbo4 with QJL | 6.1894 | +| turbo4 without QJL | 6.1756 | +| turbo3 (reference) | 6.1756 | + +QJL made quality **worse**. Removing it made turbo4 identical to turbo3. + +### Why QJL Fails for Attention + +This was independently confirmed by three groups: +- **buun (CUDA):** turbo4 PPL degrades from -0.28% at 2K to +3.69% at 64K. QJL noise accumulates with context. +- **scos-lab (independent impl):** GPT-2 b=3 MSE gives +7.6% PPL, QJL (Prod) gives +300% PPL. "QJL variance is amplified by softmax." +- **Arclabs001 (YATQ):** 3-bit MSE-only top-1 71.0% vs +QJL 61.2% (-9.8%). "QJL eliminates bias but explodes variance. Softmax tolerates uniform bias, not variance." + +The TurboQuant paper's own ablation (OpenReview) confirms: MSE-only (no QJL, no two-tier channel) still beats baselines. The QJL correction is not needed. + +**Decision:** Drop QJL entirely. + +--- + +## 4. The Speed Investigation + +With QJL dropped, turbo4 matched turbo3 quality (6.1756). But decode was still 48 tok/s vs turbo3's 78. A component isolation analysis identified the bottlenecks: + +| Component | tok/s lost | % of gap | +|-----------|-----------|---------| +| 128-block FA tiling overhead | 13.7 | 33% | +| 3-bit byte-spanning unpack | 24.3 | 58% | +| QJL sign unpack + scale | 3.6 | 9% | + +### The 3-Bit Unpack Problem + +turbo4's original 3-bit index extraction: +```metal +int bit_offset = j * 3; +int byte_idx = bit_offset / 8; +int bit_pos = bit_offset % 8; +uint16_t raw = (uint16_t)xb->qs[byte_idx]; +if (byte_idx + 1 < QK_TURBO4 * 3 / 8) { + raw |= (uint16_t)xb->qs[byte_idx + 1] << 8; +} +uint8_t idx = (uint8_t)((raw >> bit_pos) & 0x7); +``` + +Per element: division, modulo, conditional second byte read, shift, mask. 128 iterations. 58% of the total decode cost. + +turbo3's byte-aligned 2+1 bit scheme: +```metal +idx = (qs[j/4] >> ((j%4)*2)) & 0x3; +if (signs[j/8] & (1<<(j%8))) idx |= 4; +``` + +Simple shifts and masks, no byte-spanning. + +**Fix:** Switch turbo4 to 2+1 bit packing (same as turbo3). Since QJL was dropped, the `signs[]` field was free for the 3rd index bit. + +**Impact:** 48.05 → 63.79 tok/s (+33%). + +### The Full-Block Cache Problem + +turbo4's dequant allocated `float cache[128]` on the stack, dequantized all 128 elements, then returned only 4. The FA vec kernel called this 32 times per block position — each call did the full 128-element unpack redundantly. + +turbo3 avoided this by directly extracting the needed elements from the 32-element block. + +**Fix:** Direct per-element extraction for turbo4, identical pattern to turbo3. + +**Impact:** 63.79 → 78.30 tok/s (+23%). Now matching turbo3. + +--- + +## 5. The 4-Bit Breakthrough + +With QJL dropped and the dequant fixed, turbo4 was functionally "turbo3 in 128-element blocks." Same quality, same speed, different block layout. The original promise of turbo4 — better quality than turbo3 — was unfulfilled. + +The fix: use the full 4 bits for PolarQuant instead of splitting 3+1 (PolarQuant+QJL). 16 optimal centroids instead of 8. + +### Computing 4-Bit Centroids + +The optimal scalar quantizer for N(0, 1/√d) with 16 levels has centroids given by the conditional expectation within each quantile bin. For d=128: + +``` +[-0.1739, -0.1172, -0.0895, -0.0688, -0.0513, -0.0356, -0.0210, -0.0069, + 0.0069, 0.0210, 0.0356, 0.0513, 0.0688, 0.0895, 0.1172, 0.1739] +``` + +Packing: simple nibbles. 2 indices per byte, no byte-spanning. 4 bits per element with `(qs[j/2] >> ((j%2)*4)) & 0xF`. Even simpler than the 2+1 scheme. + +### Results + +| Config | PPL | vs q8_0 | Decode tok/s | vs q8_0 | +|--------|-----|---------|-------------|---------| +| q8_0 | 6.1109 | — | 85.71 | 1.00x | +| **turbo4 4-bit** | **6.1250** | **+0.23%** | **79.87** | **0.93x** | +| q4_0 | 6.1424 | +0.52% | 83.36 | 0.97x | +| turbo3 | 6.1756 | +1.06% | 76.84 | 0.90x | + +turbo4 4-bit has: +- Better quality than turbo3 (4.5x smaller PPL gap to q8_0) +- Better quality than q4_0 +- Better compression than q4_0 (3.76x vs 3.56x) +- Matching turbo3 decode speed +- Near-parity q8_0 prefill (96%) + +--- + +## 6. The Full Journey + +| Step | PPL | Decode tok/s | Change | +|------|-----|-------------|--------| +| Broken (shared SET_ROWS) | 679.27 | 44.43 | — | +| + Dedicated SET_ROWS | 6.1894 | 44.43 | Fixed block corruption | +| + Drop QJL | 6.1756 | 48.05 | QJL was hurting quality | +| + 2+1 bit packing | 6.1756 | 63.79 | Eliminated byte-spanning reads | +| + Direct-extract dequant | 6.1756 | 78.30 | Eliminated redundant full-block cache | +| + 4-bit PolarQuant | **6.1250** | **79.58** | 16 optimal centroids | + +Total improvement: PPL 679 → 6.13, decode 44 → 80 tok/s. + +--- + +## 7. Validation + +### Codex Review + +OpenAI Codex (gpt-5.3-codex) reviewed the final diff and caught an out-of-bounds memory access: the `block_turbo4_0` struct had `qs[48]` (sized for 3-bit packing) but 4-bit nibble packing writes to `qs[0..63]`. The struct was updated to `qs[64]`, maintaining the 68-byte block size. + +### Cross-Validation + +- buun's CUDA turbo4 (old QJL format) confirmed QJL degradation at long context +- scos-lab's independent implementation confirmed MSE > QJL for attention +- Arclabs001's YATQ analysis quantified QJL variance explosion through softmax +- The TurboQuant paper's own ablation (OpenReview) confirms MSE-only beats baselines + +### Long-Context Stability + +| Context | turbo4 4-bit | turbo3 | turbo4 vs turbo3 | +|---------|-------------|--------|-----------------| +| 512 | 6.1250 | 6.1756 | turbo4 wins | +| 8K | 5.5546 | 5.5700 | turbo4 wins | +| 16K | 5.0813 | 5.0630 | ~tied (within noise) | + +No degradation trend — unlike buun's QJL turbo4 which degraded from -0.28% at 2K to +3.69% at 64K. + +--- + +## 8. What We Learned + +1. **QJL is harmful for KV cache attention.** The 1-bit correction eliminates bias but explodes variance. Softmax amplifies variance. Three independent groups confirmed this. + +2. **Byte-spanning bit extraction is catastrophically expensive on GPU.** The 3-bit packed format cost 58% of total decode time. Simple nibble packing or byte-aligned 2+1 schemes are dramatically faster. + +3. **Full-block dequant caching doesn't work when the block is 128 elements.** The FA kernel calls dequant 32x per position — each call doing a full 128-element unpack and returning 4 elements. Direct extraction eliminates this 32x redundancy. + +4. **More centroids > error correction.** 16 optimal PolarQuant centroids give better quality than 8 centroids + QJL correction, at the same bit rate, with simpler implementation and faster execution. + +5. **The original paper's design was suboptimal for this application.** TurboQuant was designed for general vector quantization. In the specific context of KV cache attention (where softmax amplifies variance), the QJL component is counterproductive. + +--- + +## 9. Current Status + +turbo4 4-bit PolarQuant is on the `experiment/turbo4-quality-investigation` branch. All changes are in Metal shaders only — no upstream dependencies. + +### Prefill Context Scaling + +| Context | turbo4 tok/s | turbo3 tok/s | q8_0 tok/s | turbo4/q8_0 | +|---------|-------------|-------------|-----------|------------| +| 2K | 2682 | 2708 | 2665 | 1.01x | +| 4K | 2370 | 2289 | 2255 | 1.05x | +| 8K | 2041 | 2054 | 2002 | 1.02x | +| 16K | 1621 | 1698 | 1605 | 1.01x | +| 32K | 1141 | 1204 | 1098 | 1.04x | + +turbo4 prefill matches or exceeds q8\_0 at all context lengths. Compressed cache moves less data over the memory bus. + +### Summary + +**What works:** +- PPL: +0.23% vs q8_0 (best quantized KV cache quality we've tested) +- Decode: 93% of q8_0 (matching turbo3) +- Prefill: 101-105% of q8_0 across all context lengths +- Sparse V: active and compatible +- Long context: stable (no QJL degradation) + +### Head-Dim-Dependent Centroids (Discovery) + +buun tested our 4-bit centroids on CUDA and found they work better on hd128 but slightly worse on hd256. The reason: our centroids are Lloyd-Max optimal for N(0, 1/√d) at d=128, not standard N(0,1). + +``` +Ours (d=128, σ=0.0884): outer centroid = 0.1739 (1.97σ) +buun's (N(0,1) scaled): outer centroid = 0.2416 (2.73σ) +``` + +The post-WHT distribution is N(0, 1/√d), which depends on head_dim. Optimal centroids should be computed per head_dim: +- d=128: σ = 1/√128 ≈ 0.0884 +- d=256: σ = 1/√256 = 0.0625 + +buun confirmed: our d=128 centroids give **better quality than q8_0 on hd128** (!) but regress slightly on hd256. Head-dim-dependent codebooks may be the fix for the hd128 quality gap the entire community has been chasing. + +**What's next:** +- KLD and NIAH characterization +- Head-dim-dependent centroid codebooks (d=128 and d=256) +- Asymmetric K/V (turbo3-K + turbo4-V — blocked by cross-type FA kernel instantiation) +- Upstream integration +- CUDA port + +--- + +## Acknowledgments + +- **@spiritbuun** — CUDA turbo4 data that motivated the investigation, "do not bother" verdict that we proved could be reversed +- **scos-lab** — Independent MSE > QJL finding, K/V norm disparity data +- **Arclabs001/YATQ** — Quantified QJL variance explosion through softmax +- **OpenAI Codex** — Caught the out-of-bounds struct bug +- **Sean Rasch** — turbo4 Python test coverage (PR #41) diff --git a/docs/sparse-v-upstream-validation.md b/docs/sparse-v-upstream-validation.md new file mode 100644 index 0000000..9aca391 --- /dev/null +++ b/docs/sparse-v-upstream-validation.md @@ -0,0 +1,40 @@ +# Sparse V Upstream Validation Note + +## What changed +A single conditional added to the Metal flash attention vec kernel's V accumulation loop. When the post-softmax attention weight for a position is below 1e-6, the V dequantization and accumulation for that position is skipped. + +Gated by compile-time preprocessor define `GGML_METAL_FA_SKIP_V`, controlled via environment variable. Default: off. + +## Why safe +- The conditional only fires in the quantized V path (dequant branch) +- Threshold 1e-6 is far below meaningful contribution to output +- When disabled (`GGML_METAL_FA_SKIP_V=0`), code path is identical to unpatched +- No changes to attention scores, softmax, normalization, or K computation + +## What was tested +- PPL: 32K context, 50 chunks, wikitext-103, q8_0/q4_0/turbo3 — ON/OFF delta 0.0000 in all formats +- PPL: 8K/16K/32K multi-context sweep — ON/OFF delta 0.0000 at all depths +- Decode speed: q8_0 +5% short, turbo3 +22.8% at 32K, q4_0 neutral +- NIAH retrieval: identical or improved across formats +- KL divergence: measured, consistent with underlying quantization +- Threshold ablation: 1e-4 through 1e-8, PPL identical at all values +- Dense model: no regression +- Hardware: Apple M5 Max 128GB + +## Supported claims (safe for PR) +- No measurable PPL change when enabled (across tested formats and contexts) +- Decode speed improvement scales with V dequant cost +- Safe to enable for quantized KV cache formats on Metal + +## Unsupported claims (do NOT put in PR) +- Universal improvement across all hardware +- CUDA validation (not tested on upstream CUDA path) +- "Improves quality" (NIAH improvement is secondary signal, not primary claim) +- Specific percentage claims beyond tested setup + +## Limitations +- Validated on Apple Metal only (M5 Max) +- Threshold is hardcoded (not adaptive) +- Skip rate directly measured on smaller model (1.7B), estimated on larger +- Short context benefit is minimal (attention is dense) +- Dense models see negligible benefit (FFN dominates) diff --git a/docs/threshold-ablation-logs/asymmetric_kv_blocked.txt b/docs/threshold-ablation-logs/asymmetric_kv_blocked.txt new file mode 100644 index 0000000..6582715 --- /dev/null +++ b/docs/threshold-ablation-logs/asymmetric_kv_blocked.txt @@ -0,0 +1,26 @@ +Asymmetric K/V Testing — BLOCKED +================================== + +Date: 2026-03-28 +Branch: experiment/turbo4-quality-investigation + +Attempted: -ctk turbo3 -ctv turbo4 + +Result: KV cache allocates correctly (K turbo3: 4.38 MiB, V turbo4: 5.31 MiB) +but inference hangs at warmup. FA vec kernel has no cross-type instantiation. + +Current instantiations (same K/V type only): + turbo3 K + turbo3 V: ✅ + turbo4 K + turbo4 V: ✅ + turbo3 K + turbo4 V: ❌ (no kernel) + turbo4 K + turbo3 V: ❌ (no kernel) + +Fix needed: Add cross-type FA kernel template instantiations. +The kernel template already supports separate K and V types — just needs +the host_name + template lines for mixed configurations. + +buun's CUDA fork likely has these since he tested turbo3-K + turbo4-V +and got -0.28% vs q8_0. + +Priority: Medium — valuable for quality optimization but needs ~20 new +kernel instantiations (all K/V type combos × all dk/dv sizes). diff --git a/docs/threshold-ablation-logs/turbo4_2plus1_packing_fix.txt b/docs/threshold-ablation-logs/turbo4_2plus1_packing_fix.txt new file mode 100644 index 0000000..5c4df6f --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_2plus1_packing_fix.txt @@ -0,0 +1,37 @@ +Turbo4 2+1 Bit Packing Fix — +33% Decode Speed +================================================= + +Date: 2026-03-28 +Branch: experiment/turbo4-quality-investigation +Hardware: Apple M5 Max 128GB +Model: Qwen3.5-35B-A3B Q8_0 (MoE) + +Change: Replaced byte-spanning 3-bit packed index extraction with +turbo3-style 2+1 bit scheme (low 2 bits in qs[], high 1 bit in signs[]). +Also dropped QJL entirely (signs[] now stores index high bit, not QJL signs). + +Results: + Config | PPL | Decode tok/s | vs q8_0 + q8_0 | 6.1109 | 86.01 | 1.00x + turbo3 | 6.1756 | 78.24 | 0.91x + turbo4 2+1 bit (NEW) | 6.1756 | 63.79 | 0.74x + turbo4 no QJL (old 3-bit pack) | 6.1756 | 48.05 | 0.56x + turbo4 full (old, with QJL) | 6.1894 | 44.43 | 0.52x + +Improvement: 48.05 → 63.79 tok/s (+33% from packing fix alone) +PPL: identical to turbo3 (6.1756) + +Remaining gap vs turbo3 (63.79 vs 78.24 = 18% slower): + Entirely due to 128-block FA tiling overhead (nl=8 vs nl=2). + turbo4 block is 128 elements → dequant called 4x more per position. + The per-thread block cache mitigates this but doesn't eliminate it. + +Updated component breakdown: + Component | tok/s lost | % of gap + 128-block FA tiling overhead | 13.7 | 62% + 2+1 bit unpack + centroid LUT | 8.4 | 38% + QJL (removed) | 0.0 | 0% + TOTAL gap vs q8_0 | 22.2 | 100% + +The 3-bit byte-spanning unpack was the single biggest fixable bottleneck. +Remaining gap is structural (block size in FA kernel). diff --git a/docs/threshold-ablation-logs/turbo4_4bit_long_context.txt b/docs/threshold-ablation-logs/turbo4_4bit_long_context.txt new file mode 100644 index 0000000..d242779 --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_4bit_long_context.txt @@ -0,0 +1,33 @@ +Turbo4 4-bit PolarQuant — Long Context Validation +=================================================== + +Date: 2026-03-28 +Branch: experiment/turbo4-quality-investigation +Hardware: Apple M5 Max 128GB +Model: Qwen3.5-35B-A3B Q8_0 (MoE) + +Context scaling (turbo4 4-bit vs turbo3 3-bit): + +| Context | turbo4 4-bit | turbo3 3-bit | q8_0 | turbo4 vs turbo3 | +|---------|-------------|-------------|------|-----------------| +| 512 | 6.1250 | 6.1756 | 6.1109 | -0.08% (better) | +| 8K | 5.5546 | 5.5700 | — | -0.28% (better) | +| 16K | 5.0813 | 5.0630 | 5.0008 | +0.36% (worse) | + +turbo4 wins at short-medium context, turbo3 wins at 16K. +Differences are small (0.04-0.05 PPL) and within noise for 8-chunk runs. +Crucially: NO degradation trend — unlike buun's QJL turbo4 which degraded +from -0.28% at 2K to +3.69% at 64K. Our 4-bit approach is stable. + +Decode speed at 32K: + turbo4 4-bit: 79.36 tok/s (short), 1042.79 tok/s (pp32768+tg128) + turbo3: 78.24 tok/s (short) + +turbo4 prefill at 32K now works (1042.79 tok/s) — previously stuck/crashed. +The 4-bit nibble dequant is much faster than the old 3-bit byte-spanning path. + +Half-precision LUT test: + PPL: 6.1250 (identical) + Decode: 79.06 tok/s (within noise of 79.58 without) + Half LUT doesn't help on M5 Max — constant cache not the bottleneck at 16 entries. + May help on M1/M2 where constant cache is more constrained. diff --git a/docs/threshold-ablation-logs/turbo4_4bit_polarquant_results.txt b/docs/threshold-ablation-logs/turbo4_4bit_polarquant_results.txt new file mode 100644 index 0000000..01503b1 --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_4bit_polarquant_results.txt @@ -0,0 +1,32 @@ +Turbo4 4-bit PolarQuant — Quality Breakthrough +================================================= + +Date: 2026-03-28 +Branch: experiment/turbo4-quality-investigation +Hardware: Apple M5 Max 128GB +Model: Qwen3.5-35B-A3B Q8_0 (MoE) + +Change: Replace 3-bit PolarQuant + 1-bit QJL with 4-bit PolarQuant (16 centroids). +Optimal centroids computed analytically for N(0, 1/sqrt(128)). +Nibble packing: 2 indices per byte, no byte-spanning reads. + +Results: + Config | PPL | vs q8_0 | Decode tok/s | vs q8_0 + q8_0 | 6.1109 | — | 86.01 | 1.00x + turbo4 4-bit (NEW) | 6.1250 | +0.23% | 79.58 | 0.93x + turbo3 3-bit | 6.1756 | +1.06% | 78.24 | 0.91x + q4_0 | 6.1424 | +0.52% | 83.36 | 0.97x + +turbo4 4-bit beats turbo3 on quality by 4.5x smaller PPL gap. +turbo4 4-bit is closer to q8_0 than q4_0. +turbo4 4-bit is slightly faster than turbo3 (79.58 vs 78.24). + +FULL TURBO4 OPTIMIZATION JOURNEY: + 44.43 tok/s, PPL 679 → SET_ROWS fix + 44.43 tok/s, PPL 6.1894 → drop QJL + 48.05 tok/s, PPL 6.1756 → 2+1 bit packing + 63.79 tok/s, PPL 6.1756 → direct-extract dequant + 78.30 tok/s, PPL 6.1756 → 4-bit PolarQuant (16 centroids) + 79.58 tok/s, PPL 6.1250 ← FINAL + +Total: 44.43 → 79.58 tok/s (+79%), PPL 679 → 6.125 diff --git a/docs/threshold-ablation-logs/turbo4_component_isolation.txt b/docs/threshold-ablation-logs/turbo4_component_isolation.txt new file mode 100644 index 0000000..9f5be06 --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_component_isolation.txt @@ -0,0 +1,59 @@ +Turbo4 Component Isolation — Decode Speed Breakdown +===================================================== + +Date: 2026-03-28 +Branch: experiment/turbo4-quality-investigation +Hardware: Apple M5 Max 128GB +Model: Qwen3.5-35B-A3B Q8_0 (MoE) +Test: llama-bench, tg128 (short context decode) + +Method: Strip turbo4 dequant layer by layer, rebuild, bench. + +Results: + q8_0 baseline: 86.01 tok/s (1.00x) + turbo3 baseline: 78.24 tok/s (0.91x) + D. turbo4 no-op dequant: 72.33 tok/s (0.84x) — memory/tiling only + B. turbo4 no QJL: 48.05 tok/s (0.56x) — PolarQuant only + A. turbo4 full: 44.43 tok/s (0.52x) — PolarQuant + QJL + +Cost Breakdown: + Component | tok/s lost | % of gap + 128-block FA tiling overhead | 13.7 | 33% + 3-bit unpack + centroid lookup | 24.3 | 58% + QJL sign unpack + scale | 3.6 | 9% + TOTAL gap vs q8_0 | 41.6 | 100% + +Analysis: + 1. THE KILLER IS 3-BIT UNPACK (58% of cost) + turbo4 uses packed 3-bit indices spanning byte boundaries: + bit_offset = j * 3; byte_idx = bit_offset / 8; bit_pos = bit_offset % 8; + raw = qs[byte_idx]; if (bit_pos > 5) raw |= qs[byte_idx+1] << 8; + This is dramatically more expensive than turbo3's byte-aligned 2+1 scheme: + idx = (qs[j/4] >> ((j%4)*2)) & 0x3; if (signs[j/8] & (1<<(j%8))) idx |= 4; + + 2. 128-BLOCK TILING OVERHEAD (33% of cost) + turbo4 uses nl=8 (128 elements / 16 per dequant call). + turbo3 uses nl=2 (32 elements / 16 per dequant call). + FA calls dequant 4x more per position for turbo4. + Even with per-thread block caching, the overhead is significant. + + 3. QJL IS PURE WASTE (9% speed cost, negative quality) + QJL adds 3.6 tok/s overhead and makes PPL 0.22% WORSE. + Removing it saves speed AND improves quality. No reason to keep it. + +Comparison with turbo3: + turbo3 dequant is 78.24 tok/s (0.91x q8_0) + turbo4 no-op is 72.33 tok/s (0.84x q8_0) + Even with ZERO dequant work, turbo4's format is 7.6% slower than turbo3 + due to the 128-block tiling overhead alone. + + turbo4 with full dequant (44.43) is 43% slower than turbo3 (78.24). + The 3-bit unpack is responsible for most of this gap. + +Recommendations: + 1. turbo4 as-is is not viable for production + 2. If turbo4 is kept, it needs: + a. Drop QJL (saves speed + improves quality) + b. Redesign 3-bit packing to be byte-aligned (biggest win) + c. Reduce nl from 8 to 2 somehow (block caching improvement?) + 3. Alternative: 4-bit PolarQuant with byte-aligned packing (new format) diff --git a/docs/threshold-ablation-logs/turbo4_cuda_buun_data.txt b/docs/threshold-ablation-logs/turbo4_cuda_buun_data.txt new file mode 100644 index 0000000..170b0f5 --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_cuda_buun_data.txt @@ -0,0 +1,41 @@ +Turbo4 CUDA Data — buun (@spiritbuun, RTX 3090) +================================================= + +Source: X post 2026-03-28 09:50 AM +Model: Qwen3.5-27B, wikitext-2-raw + +Quality (PPL): + Context | q8_0 | turbo3 | turbo4 | turbo3 vs q8_0 | turbo4 vs q8_0 + 2K | 5.8375 | 5.8323 | 5.8186 | -0.09% | -0.28% + 32K | 7.2592 | 7.2139 | 7.2478 | -0.62% | -0.16% + 64K | 6.4804 | 6.5122 | 6.7192 | +0.49% | +3.69% + +Key finding: turbo4 wins at 2K (+0.19% vs turbo3) but LOSES at 64K (+3.08% vs turbo3) +QJL noise accumulates with context length. + +Speed: + Metric | q8_0 | turbo3 | turbo4 + Decode (tok/s) | 31.25 | 30.31 | 29.43 + Decode vs q8_0 | — | 97.0% | 94.2% + Prefill (tok/s) | 1133 | 1125 | 588 + Prefill vs q8_0 | — | 99.3% | 51.9% + +turbo4 prefill is 51.9% of q8_0 — catastrophically slow. +turbo3 prefill is 99.3% — near parity. + +buun's conclusion: "turbo3 is better in every way. Do not bother using turbo4." + +Recommended configs (buun): + Quality (hd256): -ctk turbo3 -ctv turbo3 (LA-1) + Quality (hd128, <8K ctx): -ctk turbo3 -ctv turbo4 + Quality (hd128, >8K ctx): -ctk turbo3 -ctv turbo3 + Max VRAM savings: -ctk turbo3 -ctv turbo2 + Long context (32K-128K): -ctk turbo3 -ctv turbo3 (uniform) + +Our Metal findings align: + turbo4 WITH QJL: 6.1894 PPL (+0.22% worse than turbo3) + turbo4 WITHOUT QJL: 6.1756 PPL (identical to turbo3) + QJL hurts on both CUDA and Metal. + +@jackdaniel2025 independently confirmed: "turbo4 is not worth using" +@skinbagwbones interested in collaboration on weight+KV compression stacking diff --git a/docs/threshold-ablation-logs/turbo4_dequant_then_attend_wip.txt b/docs/threshold-ablation-logs/turbo4_dequant_then_attend_wip.txt new file mode 100644 index 0000000..5d0ec20 --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_dequant_then_attend_wip.txt @@ -0,0 +1,44 @@ +Turbo4 Dequant-Then-Attend — WIP, Stashed +============================================ + +Date: 2026-03-28 +Branch: experiment/turbo4-quality-investigation (stash) + +Approach: Bulk dequant turbo4 K/V to fp16 temp buffer during prefill, +then dispatch f16 FA kernel for simdgroup matrix performance. + +What was done: +1. kernel_turbo4_dequant_f16 Metal compute shader — committed (661794f) +2. Extra tmp buffer allocation for fp16 K+V — partially done +3. FA dispatch override (strides, pipeline, buffer IDs) — incomplete, stashed + +Why stashed: +- FA dispatch code uses GGML_TENSOR_LOCALS (const) for strides +- Shadowing all stride variables cascades through padding, blk, and FA dispatches +- Two identical padding arg structs in non-vec path (kvpad + mask) need same override +- Pipeline lookup needs manual name construction (type override) +- Risk of breaking the entire FA path for 13% prefill gain + +Codex caught 6 issues in the dispatch code: +1. CRITICAL: V buffer sized using K dims (FIXED in stash) +2. HIGH: Pipeline fallback reads uninitialized data (FIXED) +3. HIGH: Stride mismatch after buffer swap (PARTIALLY FIXED — stashed) +4. MEDIUM: Block count floor division (FIXED with assert) +5. MEDIUM: uint32 overflow risk (noted) +6. LOW: Unused turbo4_dequanted flag (addressed) + +Prefill ceiling test: + turbo4 4-bit: 2037 tok/s (96% of q8_0) + f16 ceiling: 2314 tok/s (109% of q8_0) + Maximum gain: ~13% + +Recommendation: + Implement as a graph-level ggml_cast(turbo4 → f16) operation instead + of modifying FA dispatch. This avoids touching the fragile FA dispatch + code and can be inserted cleanly in llama-graph.cpp before the FA op. + + Requires: ggml_metal_op_cast support for TURBO4_0 → F16 (new op). + +Alternative: Upstream llama.cpp may add a general "need_f16_K/V" flag + (apollosenvy's ROCm PR used this approach). Wait for that infrastructure + and plug turbo4 dequant into it. diff --git a/docs/threshold-ablation-logs/turbo4_direct_extract_results.txt b/docs/threshold-ablation-logs/turbo4_direct_extract_results.txt new file mode 100644 index 0000000..26fed00 --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_direct_extract_results.txt @@ -0,0 +1,35 @@ +Turbo4 Direct-Extract Dequant — Matches Turbo3 Speed +====================================================== + +Date: 2026-03-28 +Branch: experiment/turbo4-quality-investigation +Hardware: Apple M5 Max 128GB +Model: Qwen3.5-35B-A3B Q8_0 (MoE) + +Change: Replaced full-block cache dequant with direct per-element extraction. +Old: allocate float cache[128], dequant all 128 elements, return 4 (called 32x per block) +New: directly extract needed elements using 2+1 bit packing (identical to turbo3 pattern) + +FULL OPTIMIZATION JOURNEY: + Config | PPL | Decode tok/s | vs q8_0 + q8_0 baseline | 6.1109 | 86.01 | 1.00x + turbo3 baseline | 6.1756 | 78.24 | 0.91x + turbo4 original (broken) | 6.1894 | 44.43 | 0.52x + + SET_ROWS fix | 6.1894 | 44.43 | 0.52x + + drop QJL | 6.1756 | 48.05 | 0.56x + + 2+1 bit packing | 6.1756 | 63.79 | 0.74x + + direct-extract dequant (FINAL) | 6.1756 | 78.30 | 0.91x + +Total improvement: 44.43 → 78.30 tok/s (+76%) +PPL: identical to turbo3 (6.1756) +Speed: identical to turbo3 (78.30 vs 78.24, within noise) + +Root causes fixed: + 1. SET_ROWS used turbo3 packing for turbo4 blocks (PPL 679 → 6.19) + 2. QJL correction hurts quality, not helps (PPL 6.19 → 6.18) + 3. 3-bit byte-spanning unpack (48 → 64 tok/s, +33%) + 4. Full-block cache redundancy (64 → 78 tok/s, +23%) + + Items 3+4 together: 48 → 78 tok/s (+63%) + +turbo3 sanity: 6.1756, 78.24 tok/s (unchanged, no regression) diff --git a/docs/threshold-ablation-logs/turbo4_full_characterization.txt b/docs/threshold-ablation-logs/turbo4_full_characterization.txt new file mode 100644 index 0000000..888e55e --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_full_characterization.txt @@ -0,0 +1,92 @@ +Turbo4 4-bit PolarQuant — Full Characterization +================================================== + +Date: 2026-03-28 +Branch: experiment/turbo4-quality-investigation @ 661794f +Hardware: Apple M5 Max 128GB +Model: Qwen3.5-35B-A3B Q8_0 (MoE) +Sparse V: enabled (auto on M5) + +=== PPL (wikitext-2, 512c, 8 chunks) === + +| Cache | PPL | vs q8_0 | Bits/val | Compression | +|-------|-----|---------|----------|-------------| +| q8_0 | 6.1109 | — | 8.5 | 1.88x | +| turbo4 | 6.1250 | +0.23% | 4.25 | 3.76x | +| q4_0 | 6.1424 | +0.52% | 4.5 | 3.56x | +| turbo3 | 6.1756 | +1.06% | 3.5 | 4.57x | + +=== PREFILL CONTEXT SCALING (tok/s) === + +| Context | turbo4 | turbo3 | q8_0 | turbo4/q8_0 | turbo3/q8_0 | +|---------|--------|--------|------|------------|------------| +| 2K | 2682 | 2708 | 2665 | 1.01x | 1.02x | +| 4K | 2370 | 2289 | 2255 | 1.05x | 1.01x | +| 8K | 2041 | 2054 | 2002 | 1.02x | 1.03x | +| 16K | 1621 | 1698 | 1605 | 1.01x | 1.06x | +| 32K | 1141 | 1204 | 1098 | 1.04x | 1.10x | + +=== DECODE (tok/s) === +NOTE: Initial run affected by battery throttling. Rerun below on AC power. + +Clean rerun (plugged in): +| Config | tg128 | pp32768+tg128 | vs q8_0 | +|--------|-------|---------------|---------| +| q8_0 | 85.71 ± 0.81 | 1173.91 ± 5.41 | — | +| turbo4 | 79.87 ± 0.16 | 1060.12 ± 7.04 | 0.93x | +| turbo3 | 76.84 ± 1.02 | 1141.74 ± 13.59 | 0.90x | + +turbo4 is FASTER than turbo3 on decode (79.87 vs 76.84). +Nibble packing + direct-extract dequant is simpler and faster than +turbo3's 2+1 bit scheme + constant-memory centroid LUT. + +=== LONG-CONTEXT PPL (wikitext-2, 8 chunks) === + +| Context | turbo4 | turbo3 | q8_0 | turbo4 vs q8_0 | +|---------|--------|--------|------|---------------| +| 512 | 6.1250 | 6.1756 | 6.1109 | +0.23% | +| 8K | 5.5546 | 5.5700 | 5.5093 | +0.82% | +| 16K | 5.0813 | 5.0630 | 5.0468 | +0.68% | + +=== KLD vs f16 (wikitext-2, 512c, 8 chunks) === + +| Cache | Mean KLD | Δp RMS | Same top-p % | +|-------|----------|--------|-------------| +| q8_0 | 0.001549 | 1.23% | 98.43% | +| turbo4 | 0.009633 | 2.71% | 95.98% | +| q4_0 | 0.008091 | 2.75% | 95.83% | +| turbo3 | 0.016145 | 4.09% | 94.31% | + +turbo4 KLD is better than turbo3 (0.0096 vs 0.0161). +turbo4 same-top-p matches q4_0 exactly (95.98% vs 95.83%). + +=== NIAH (single needle, seed 42, 3 depths × 3 positions) === + +turbo4: +| Depth | 4K | 8K | 16K | +|--------|------|------|------| +| 0 % | FAIL | PASS | PASS | +| 50 % | PASS | PASS | PASS | +| 100 % | PASS | FAIL | PASS | + +Score: 7/9 + +Reference: + q8_0: 7/9 (same score) + turbo3 + sparse V: 9/9 + turbo3 (no sparse V): 7/9 + +=== 70-PAGE PDF SERVER BENCHMARK (~24K context) === + +| Config | Prefill tok/s | Decode tok/s | +|--------|-------------|-------------| +| q8_0 | 1449.9 | 68.2 | +| turbo4 | 1405.9 | 63.7 | +| turbo3 | 1417.8 | 53.3 | + +turbo4 beats turbo3 by 20% on real-world server decode (63.7 vs 53.3). +turbo4 server decode is 93% of q8_0 (63.7 / 68.2). + +Commands used: + llama-perplexity ... -ctk turbo4 -ctv turbo4 -fa on --chunks 8 -ngl 99 + llama-bench ... -ctk turbo4 -ctv turbo4 -fa 1 -ngl 99 diff --git a/docs/threshold-ablation-logs/turbo4_prefill_comparison.txt b/docs/threshold-ablation-logs/turbo4_prefill_comparison.txt new file mode 100644 index 0000000..fc6a02b --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_prefill_comparison.txt @@ -0,0 +1,27 @@ +Turbo4 4-bit Prefill Speed Comparison +====================================== + +Date: 2026-03-28 +Hardware: Apple M5 Max 128GB +Model: Qwen3.5-35B-A3B Q8_0 (MoE) +Branch: experiment/turbo4-quality-investigation + +Prefill at 8K context: + f16 (ceiling): 2313.58 tok/s (1.09x q8_0) + q8_0 (baseline): 2123.03 tok/s (1.00x) + turbo4 4-bit: 2037.31 tok/s (0.96x q8_0) + +turbo4 prefill is 96% of q8_0 — near parity. +The f16 ceiling is only 13% above turbo4. + +Compared to buun's CUDA turbo4 (old QJL format): + buun: 588 tok/s (51.9% of q8_0) — catastrophically slow + Us: 2037 tok/s (96% of q8_0) — near parity + +The massive CUDA prefill regression was caused by the old 3-bit +byte-spanning dequant + QJL overhead. Our 4-bit nibble packing +with direct-extract dequant eliminates both bottlenecks. + +Dequant-then-attend approach would give at most 2313 tok/s (+13%). +Not worth the infrastructure complexity given current near-parity. +Deprioritized — focus on other wins. diff --git a/docs/threshold-ablation-logs/turbo4_qjl_ablation.txt b/docs/threshold-ablation-logs/turbo4_qjl_ablation.txt new file mode 100644 index 0000000..ce6e983 --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_qjl_ablation.txt @@ -0,0 +1,41 @@ +Turbo4 QJL Ablation — QJL Hurts Quality +======================================== + +Date: 2026-03-28 +Branch: experiment/turbo4-quality-investigation +Hardware: Apple M5 Max 128GB +Model: Qwen3.5-35B-A3B Q8_0 +Corpus: wikitext-2 wiki.test.raw, c=512, 8 chunks + +Experiment: Disable QJL correction in turbo4 dequant to isolate its effect. +Changed line 776 from: + cache[i] = (recon[i] + signs_f[i] * qjl_scale) * norm; +To: + cache[i] = recon[i] * norm; // PolarQuant only + +Results: + q8_0: 6.1109 (baseline) + turbo3: 6.1756 (+1.06%) + turbo4 WITH QJL: 6.1894 (+1.28%) + turbo4 WITHOUT QJL: 6.1756 (+1.06%) ← same as turbo3 + +Analysis: + QJL correction INCREASES perplexity by 0.22%. + Removing QJL makes turbo4 PPL identical to turbo3. + + This confirms scos-lab's finding (issue #45): + - QJL variance is amplified by softmax + - MSE (PolarQuant alone) beats Prod (PolarQuant + QJL) for attention + - The 1-bit QJL correction adds noise, not signal, in the KV cache context + +Implication: + turbo4 as designed (3-bit PolarQuant + 1-bit QJL) is strictly worse than + turbo3 (3-bit PolarQuant) for KV cache. The QJL bit is wasted. + + Options: + 1. Drop QJL, make turbo4 = "turbo3 in 128-block format" (same quality, different block layout) + 2. Use the extra bit for 4-bit PolarQuant (more centroids) instead of QJL + 3. Keep turbo4 name but implement as 4-bit MSE-only (no QJL) + + Option 3 is most promising — 4-bit PolarQuant should beat both turbo3 + and q4_0 since PolarQuant is theoretically optimal at any bit rate. diff --git a/docs/threshold-ablation-logs/turbo4_setrows_fix_results.txt b/docs/threshold-ablation-logs/turbo4_setrows_fix_results.txt new file mode 100644 index 0000000..37b47de --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_setrows_fix_results.txt @@ -0,0 +1,36 @@ +Turbo4 SET_ROWS Fix Results +=========================== + +Date: 2026-03-28 +Branch: experiment/turbo4-quality-investigation +Hardware: Apple M5 Max 128GB +Model: Qwen3.5-35B-A3B Q8_0 +Corpus: wikitext-2 wiki.test.raw, c=512, 8 chunks + +Fix: Dedicated kernel_set_rows_turbo4 with correct 3-bit packing + QJL signs. +Replaces shared kernel_set_rows_turbo which used turbo3's 2+1 bit scheme. + +BEFORE FIX (shared SET_ROWS, turbo3 packing applied to turbo4): + turbo4 PPL = 679.27 (completely broken) + +AFTER FIX (dedicated SET_ROWS with correct 3-bit packing + QJL): + turbo4 PPL = 6.1894 +/- 0.331 + +COMPARISON: + q8_0: 6.1109 (baseline) + turbo3: 6.1756 (+1.06% vs q8_0) + turbo4: 6.1894 (+1.28% vs q8_0, +0.22% vs turbo3) + +OBSERVATION: + turbo4 (4.25 bits) is WORSE than turbo3 (3.5 bits) by 0.22%. + This is unexpected — more bits should mean better quality. + Possible cause: QJL correction adding noise instead of signal. + Consistent with scos-lab finding (issue #45): QJL variance + amplified by softmax, MSE beats Prod for attention. + +NEXT STEPS: + 1. Test turbo4 with QJL disabled (zero out signs) — isolate QJL effect + 2. If QJL hurts, turbo4 becomes "turbo3 in 128-block format" + 3. Run decode speed comparison + +turbo3 sanity: 6.1756 (unchanged, no regression from turbo4 fix) diff --git a/docs/threshold-ablation-logs/turbo4_vs_buun_comparison.txt b/docs/threshold-ablation-logs/turbo4_vs_buun_comparison.txt new file mode 100644 index 0000000..90be7a4 --- /dev/null +++ b/docs/threshold-ablation-logs/turbo4_vs_buun_comparison.txt @@ -0,0 +1,51 @@ +Turbo4 4-bit PolarQuant vs buun's CUDA turbo4 (QJL) +===================================================== + +Date: 2026-03-28 +Our setup: Apple M5 Max 128GB, Qwen3.5-35B-A3B Q8_0 (MoE) +buun's setup: RTX 3090 24GB, Qwen3.5-27B (dense) + +Note: different models and hardware. Absolute PPL values differ. +Compare relative gaps vs q8_0 within each setup. + +QUALITY (PPL, lower is better): + +Context | buun turbo4 (QJL) | buun vs q8_0 | Us turbo4 (4-bit) | Us vs q8_0 +512 | — | — | 6.1250 | +0.23% +2K | 5.8186 | -0.28% | — | — +8K | — | — | 5.5546 | +0.82% +16K | — | — | 5.0813 | +0.68% +32K | 7.2478 | -0.16% | — | — +64K | 6.7192 | +3.69% !! | expected stable | no QJL + +buun turbo3 reference: +2K: 5.8323 (-0.09%) 32K: 7.2139 (-0.62%) 64K: 6.5122 (+0.49%) + +Us turbo3 reference: +512: 6.1756 (+1.06%) 8K: 5.5700 (+1.10%) 16K: 5.0630 (+0.32%) + +SPEED: + +Metric | buun CUDA turbo4 | buun CUDA turbo3 | Us Metal turbo4 | Us Metal turbo3 +Decode tok/s | 29.43 | 30.31 | 79.58 | 78.24 +Decode vs q8_0 | 94.2% | 97.0% | 92.5% | 90.9% +Prefill tok/s | 588 | 1125 | 1042.79 | ~2747 (512c) +Prefill vs q8_0 | 51.9% | 99.3% | — | ~102% + +q8_0 baselines: + buun: 31.25 decode, 1133 prefill + Us: 86.30 decode, — prefill + +KEY DIFFERENCES: +1. Our turbo4 drops QJL entirely → no long-context degradation (buun's +3.69% at 64K) +2. Our turbo4 uses 4-bit PolarQuant (16 centroids) → better quality at short/medium context +3. Our prefill works (1042 tok/s) vs buun's is catastrophic (51.9%) +4. buun's CUDA decode ratio is higher (94.2% vs 92.5%) — register LUT advantage on CUDA +5. buun tested on dense 27B, we test on MoE 35B — different attention/FFN balance + +CONCLUSION: +Our 4-bit PolarQuant approach is strictly better than the paper's original turbo4 design: +- Better quality (no QJL noise) +- Better speed (nibble packing + direct extract) +- Stable at long context (no degradation) +- Simpler implementation (no QJL matrices, no residual computation) diff --git a/docs/turbo4-investigation.md b/docs/turbo4-investigation.md new file mode 100644 index 0000000..4d7af41 --- /dev/null +++ b/docs/turbo4-investigation.md @@ -0,0 +1,144 @@ +# Turbo4 Quality Investigation + +**Date:** 2026-03-28 +**Branch:** `experiment/turbo4-quality-investigation` +**Problem:** turbo4 PPL is 6.38 (expected ~6.1 based on 4.25 bits, should be between q4_0 and q8_0) + +## Background + +turbo4 = 3-bit PolarQuant + 1-bit QJL residual correction = 4.25 bits/value, 3.8x compression. + +The QJL (Quantized Johnson-Lindenstrauss) bit is supposed to correct the PolarQuant residual error, giving turbo4 better quality than turbo3 despite only 0.75 more bits. Without QJL working, turbo4 is essentially turbo3 with wasted space. + +## Bugs Found (7 total) + +### Tier 1 — Critical (primary cause of degradation) + +**Bug 1: Residual computed in wrong basis (Metal quantize)** +- Location: ggml-metal.metal ~line 536 +- Metal computes `residual = normalized - centroid_reconstruction` (rotated space) +- C reference computes `residual = normalized - inverse_rotated(centroid_reconstruction)` (original space) +- The residual must be in the original normalized space for QJL to work + +**Bug 2: QJL matrix multiplication missing in Metal dequant** +- Location: ggml-metal.metal ~line 773 +- Metal uses signs directly as ±1 values +- C reference applies `matvec(turbo_qjl_matrix_t, signs, qjl_recon, d)` +- Without the matrix multiplication, QJL reconstruction is random noise + +**Bug 3: SET_ROWS kernel missing QJL step entirely** +- Location: ggml-metal.metal ~line 9665 (kernel_set_rows_turbo) +- The shared SET_ROWS template only does PolarQuant (3-bit quantization) +- For turbo4, it should also: compute residual, apply QJL matrix, pack signs, store rnorm +- Currently: QJL correction is zero in the KV cache write path + +### Tier 2 — Data corruption + +**Bug 4: 3-bit packing format mismatch in SET_ROWS** +- SET_ROWS uses turbo3's 2+1 bit scheme (2 bits in qs, 1 in signs) +- turbo4 uses packed 3-bit indices (spanning byte boundaries) +- Indices are corrupted for turbo4 blocks + +**Bug 5: rnorm uninitialized in C quantizer** +- `quantize_row_turbo4_0_ref` never sets `y[block].rnorm` +- `dequantize_row_turbo4_0` reads `x[block].rnorm` for QJL scale +- QJL scale is computed from garbage memory + +**Bug 6: Metal invents rnorm from wrong-basis residual** +- Metal computes and stores rnorm, but from the wrong-basis residual (Bug 1) +- Even if rnorm were used correctly, the value is wrong + +### Tier 3 — Design issue + +**Bug 7: WHT used instead of QJL random matrix in Metal** +- Metal uses `turbo_wht_signs1/2` for QJL rotation +- C reference uses a separate `turbo_qjl_matrix` (random normal matrix) +- May be intentional optimization (WHT approximates random projection) +- But breaks exact symmetry with C reference + +## Fix Priority + +1. Fix Bug 3 (SET_ROWS) — most impactful, the KV cache write path +2. Fix Bug 1 (residual basis) — enables correct QJL in Metal quantize +3. Fix Bug 2 (QJL matrix in dequant) — enables correct QJL reconstruction +4. Fix Bug 5 (rnorm) — fix C reference, then Metal follows +5. Fix Bug 4 (3-bit packing) — correct format for turbo4 blocks +6. Fix Bug 7 (rotation matrix) — align Metal with C reference + +## Baseline Measurements + +**On reverted main branch (shared SET_ROWS, turbo3-specific packing):** + +| Cache | PPL (c=512, 8ch, wikitext-2) | +|-------|------------------------------| +| q8_0 | 6.1109 | +| turbo3 | 6.1756 | +| turbo4 | **679.27** (completely broken) | + +**On PR #4 branch (split SET_ROWS, partial turbo4 fix):** + +| Cache | PPL | +|-------|-----| +| turbo4 | **6.38** (better but still elevated) | + +turbo4 has never worked correctly on Metal: +- Main branch: SET_ROWS uses turbo3 packing → turbo4 data corrupted → PPL 679 +- PR #4: SET_ROWS split fixed packing → PPL 6.38, but QJL still broken (bugs 1-3) +- The 6.38 is turbo4 running as "turbo3 with wasted QJL bits" — no correction applied + +## Issue #45 Insight (scos-lab) + +Independent implementation found MSE beats QJL (Prod) for attention: +- GPT-2 b=3: MSE +7.6% PPL, QJL (Prod) +300% PPL +- Reason: QJL variance amplified by softmax +- This suggests even with all bugs fixed, turbo4's QJL approach may be + fundamentally less effective than expected for KV cache attention + +## Expected Fix Impact + +Fixing bugs 1-7 should bring turbo4 to ~6.05-6.12 PPL range IF QJL works +as designed. But issue #45's finding suggests QJL may not help much for +attention specifically. Worst case: turbo4 = turbo3 quality at 4.25 bits +(more bits, same quality, less compression). Best case: ~0.5% PPL improvement +over turbo3. + +## Fix Results + +### SET_ROWS Fix (Priority 1) ✅ + +Dedicated `kernel_set_rows_turbo4` with correct 3-bit packing + QJL signs. + +| State | turbo4 PPL | +|-------|-----------| +| Before (shared turbo3 SET_ROWS) | **679.27** | +| After (dedicated turbo4 SET_ROWS) | **6.1894** | +| turbo3 reference | 6.1756 | + +PPL dropped from 679 to 6.19. turbo4 is now functional. + +### QJL Ablation (Priority 2) ✅ + +Disabled QJL in dequant (`cache[i] = recon[i] * norm` instead of `recon[i] + signs_f[i] * qjl_scale`): + +| Config | PPL | +|--------|-----| +| turbo4 WITH QJL | 6.1894 | +| turbo4 WITHOUT QJL | **6.1756** | +| turbo3 (reference) | 6.1756 | + +**QJL hurts quality.** Removing it makes turbo4 identical to turbo3. +Confirms scos-lab finding (issue #45): QJL variance amplified by softmax. + +### Conclusion + +turbo4 as designed (3-bit PolarQuant + 1-bit QJL) is strictly worse than turbo3 +for KV cache attention. The QJL bit adds noise, not signal. + +**Recommended direction for turbo4:** +1. Drop QJL entirely +2. Use all 4 bits for PolarQuant (16 centroids instead of 8) +3. 4-bit PolarQuant should beat both turbo3 and q4_0 since PolarQuant + is theoretically optimal at any bit rate + +This would make turbo4 a genuine quality upgrade over turbo3, not just +"turbo3 with a wasted correction bit." diff --git a/docs/turbo4-next-steps.md b/docs/turbo4-next-steps.md new file mode 100644 index 0000000..c09a4dc --- /dev/null +++ b/docs/turbo4-next-steps.md @@ -0,0 +1,42 @@ +# Turbo4 & Related Next Steps + +## Immediate: Fix turbo4 SET_ROWS (make it functional) + +Priority 1: Split SET_ROWS kernel so turbo4 uses correct 3-bit packing +- This alone should bring PPL from 679 → ~6.3-6.4 range +- Sean's PR #4 partially did this but introduced turbo3 regression +- Need to do it without breaking turbo3 + +Priority 2: Evaluate if QJL actually helps +- scos-lab (issue #45) found QJL (Prod) is worse than MSE for attention +- QJL variance amplified by softmax → may not improve turbo4 over turbo3 +- Test: turbo4 with QJL disabled (just 3-bit PolarQuant in 128-block format) +- If no improvement, turbo4 should be "turbo3 in 128-block format" not "turbo3 + QJL" + +Priority 3: Fix QJL bugs (if QJL is worth keeping) +- Bug 1: Residual in wrong basis +- Bug 2: QJL matrix missing in dequant +- Bug 7: Wrong rotation matrices +- Only worth fixing if Priority 2 shows QJL helps + +## Backlog: RotorQuant Review + +johndpope posted RotorQuant — Clifford algebra variant claiming faster rotation via Cl(3,0) rotors. +- Repo: https://github.com/scrya-com/rotorquant (issue #30 on our repo) +- Key concern: rotation isn't our bottleneck (<1% of inference) +- No PPL numbers provided +- Worth reviewing the math to see if the Clifford approach has other advantages +- See also: [[RotorQuant - Clifford Algebra KV Cache Compression]] in Obsidian + +## Backlog: Attention-Gated Optimizations + +From docs/attention-gated-optimizations.md: +- #3 O rescaling skip — TODO +- #4 Softmax exp() skip — TODO (most promising remaining) + +## Backlog: Community Issues + +- sabotage3d CUDA bugs (llama-cpp-turboquant #14) — follow up by Apr 2 +- Vulkan device support (turboquant_plus #46) +- M4 crash during KV cache init (turboquant_plus #35) +- scos-lab findings on K/V norm disparity (turboquant_plus #45) — worth investigating for asymmetric K/V diff --git a/docs/turbo4-profiling-plan.md b/docs/turbo4-profiling-plan.md new file mode 100644 index 0000000..c2f1f51 --- /dev/null +++ b/docs/turbo4-profiling-plan.md @@ -0,0 +1,76 @@ +# Turbo4 Systematic Profiling Plan + +Goal: Explain exactly WHY turbo4 is worse than turbo3, with numbers at each level. Same rigor as the 14 decode experiments that led to sparse V. + +## Phase 1: Component Isolation (Metal, M5 Max) + +Strip turbo4 dequant layer by layer, measure each: + +| Experiment | What's disabled | Measures | +|-----------|----------------|----------| +| A. Full turbo4 (baseline) | Nothing | Total cost | +| B. QJL disabled | `signs_f[i] * qjl_scale` zeroed | QJL correction cost | +| C. QJL + rotation disabled | Skip inverse WHT in dequant | Rotation cost | +| D. No-op dequant | Return zeros | Dequant floor (memory only) | + +For each: PPL + decode tok/s + prefill tok/s + +Expected: B should match turbo3 speed. C should be faster. D gives the floor. + +## Phase 2: Block Size Effect + +| Experiment | Config | Measures | +|-----------|--------|----------| +| E. turbo3 (4x32 blocks) | Current turbo3 | Reference | +| F. turbo4 PolarQuant-only (1x128 blocks) | turbo4 with QJL disabled | Block size overhead | + +Same compression (3-bit PolarQuant), different block layout. Isolates the 128 vs 32 block size effect on FA tiling and cache locality. + +## Phase 3: QJL Noise Accumulation Curve + +PPL at increasing context lengths to chart degradation: + +| Context | turbo3 PPL | turbo4 PPL | turbo4 no-QJL PPL | QJL noise | +|---------|-----------|-----------|-------------------|-----------| +| 512 | ? | ? | ? | ? | +| 2K | ? | ? | ? | ? | +| 8K | ? | ? | ? | ? | +| 16K | ? | ? | ? | ? | +| 32K | ? | ? | ? | ? | + +buun's CUDA data shows: +0.19% at 2K, -0.16% at 32K, +3.69% at 64K. +We need the Metal curve to compare. + +## Phase 4: Prefill Deep Dive + +turbo4 prefill is catastrophically slow (buun: 51.9% of q8_0 on CUDA). + +Root cause candidates: +1. **nl=8 vs nl=2** — non-vec FA kernel calls dequant 8x per block (turbo4) vs 2x (turbo3). 4x more dequant invocations. +2. **128-element dequant** — each dequant call processes 16 elements from a 128-element block, requiring full block read every time. +3. **Full inverse WHT per dequant call** — the block cache should prevent this, but verify. +4. **QJL matrix multiply in dequant** — even the broken version adds overhead. + +Test: profile prefill at 8K with turbo4 vs turbo3, log time per operation. + +## Phase 5: Recommendations + +Based on profiling results, determine: +1. Is turbo4-without-QJL worth keeping? (Same quality as turbo3, potentially different speed characteristics) +2. Should turbo4 become 4-bit PolarQuant? (16 centroids, no QJL, should beat turbo3 quality) +3. Should turbo4 be deprecated entirely? (buun's recommendation) +4. Is asymmetric K/V (turbo3 K, turbo4 V) worth pursuing? + +## Cross-Reference + +- buun's CUDA data: `threshold-ablation-logs/turbo4_cuda_buun_data.txt` +- scos-lab QJL finding: turboquant_plus issue #45 +- Our QJL ablation: `threshold-ablation-logs/turbo4_qjl_ablation.txt` +- Our SET_ROWS fix: `threshold-ablation-logs/turbo4_setrows_fix_results.txt` +- 7 bugs found: `turbo4-investigation.md` + +## Hardware + +- Primary: Apple M5 Max 128GB +- Cross-reference: buun's RTX 3090 24GB data +- Model: Qwen3.5-35B-A3B Q8_0 (MoE) — same as all prior benchmarks diff --git a/niah_turbo4/niah_single_20260328_190947.json b/niah_turbo4/niah_single_20260328_190947.json new file mode 100644 index 0000000..0e1f3bb --- /dev/null +++ b/niah_turbo4/niah_single_20260328_190947.json @@ -0,0 +1,170 @@ +{ + "model": "Qwen3.5-35B-A3B-Q8_0", + "mode": "single", + "seed": 42, + "timestamp": "2026-03-28T19:09:47.863363+00:00", + "results": [ + { + "mode": "single", + "context_length": 4096, + "cache_type": "turbo4", + "needle_depth_pct": 0.0, + "needle_count": 1, + "accuracy_pct": 0.0, + "passed": false, + "trials": [ + { + "expected": "9976791", + "response": "", + "found": false, + "needle_depth_pct": 0.0, + "context_length": 4096 + } + ] + }, + { + "mode": "single", + "context_length": 4096, + "cache_type": "turbo4", + "needle_depth_pct": 0.5, + "needle_count": 1, + "accuracy_pct": 100.0, + "passed": true, + "trials": [ + { + "expected": "9017128", + "response": "9017128", + "found": true, + "needle_depth_pct": 0.5, + "context_length": 4096 + } + ] + }, + { + "mode": "single", + "context_length": 4096, + "cache_type": "turbo4", + "needle_depth_pct": 1.0, + "needle_count": 1, + "accuracy_pct": 100.0, + "passed": true, + "trials": [ + { + "expected": "5422861", + "response": "5422861", + "found": true, + "needle_depth_pct": 1.0, + "context_length": 4096 + } + ] + }, + { + "mode": "single", + "context_length": 8192, + "cache_type": "turbo4", + "needle_depth_pct": 0.0, + "needle_count": 1, + "accuracy_pct": 100.0, + "passed": true, + "trials": [ + { + "expected": "2255279", + "response": "2255279", + "found": true, + "needle_depth_pct": 0.0, + "context_length": 8192 + } + ] + }, + { + "mode": "single", + "context_length": 8192, + "cache_type": "turbo4", + "needle_depth_pct": 0.5, + "needle_count": 1, + "accuracy_pct": 100.0, + "passed": true, + "trials": [ + { + "expected": "3488931", + "response": "3488931", + "found": true, + "needle_depth_pct": 0.5, + "context_length": 8192 + } + ] + }, + { + "mode": "single", + "context_length": 8192, + "cache_type": "turbo4", + "needle_depth_pct": 1.0, + "needle_count": 1, + "accuracy_pct": 0.0, + "passed": false, + "trials": [ + { + "expected": "6130554", + "response": "", + "found": false, + "needle_depth_pct": 1.0, + "context_length": 8192 + } + ] + }, + { + "mode": "single", + "context_length": 16384, + "cache_type": "turbo4", + "needle_depth_pct": 0.0, + "needle_count": 1, + "accuracy_pct": 100.0, + "passed": true, + "trials": [ + { + "expected": "7073551", + "response": "7073551", + "found": true, + "needle_depth_pct": 0.0, + "context_length": 16384 + } + ] + }, + { + "mode": "single", + "context_length": 16384, + "cache_type": "turbo4", + "needle_depth_pct": 0.5, + "needle_count": 1, + "accuracy_pct": 100.0, + "passed": true, + "trials": [ + { + "expected": "4633521", + "response": "4633521", + "found": true, + "needle_depth_pct": 0.5, + "context_length": 16384 + } + ] + }, + { + "mode": "single", + "context_length": 16384, + "cache_type": "turbo4", + "needle_depth_pct": 1.0, + "needle_count": 1, + "accuracy_pct": 100.0, + "passed": true, + "trials": [ + { + "expected": "6780688", + "response": "6780688", + "found": true, + "needle_depth_pct": 1.0, + "context_length": 16384 + } + ] + } + ] +} \ No newline at end of file diff --git a/niah_turbo4/niah_single_20260328_190947.md b/niah_turbo4/niah_single_20260328_190947.md new file mode 100644 index 0000000..e487d16 --- /dev/null +++ b/niah_turbo4/niah_single_20260328_190947.md @@ -0,0 +1,11 @@ +# TurboQuant NIAH Benchmark: Qwen3.5-35B-A3B-Q8_0 +Mode: single | Seed: 42 | 2026-03-28 19:09 UTC + +## Single Needle Retrieval: turbo4 + +| Depth | 4K | 8K | 16K | +|--------|------|------|------| +| 0 % | ❌ | ✅ | ✅ | +| 50 % | ✅ | ✅ | ✅ | +| 100 % | ✅ | ❌ | ✅ | +