From 9b66d5f99ace118620081b838387f33d42f265a2 Mon Sep 17 00:00:00 2001 From: Seung-seob Lee Date: Tue, 16 Jun 2026 19:23:19 -0400 Subject: [PATCH 1/2] portable: load the Qwen 3.5 / 3.6 hybrid arch from GGUF MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The portable driver could parse a qwen35/qwen35moe GGUF's architecture but not actually load it: `build_qwen3_5_` threw "layer_types missing", and even past that the GGUF→HF tensor mapper had no linear-attn / shared-expert cases and did not invert the converter's value transforms. Only the safetensors path was ever exercised. This teaches the GGUF path the full hybrid arch. gguf_hparams.cpp: derive `layer_types` from `qwen35.full_attention_interval` (layer i is full attention iff (i+1) % interval == 0, else linear), and fill the linear-attention dims from the `qwen35.ssm.*` keys (group_count→num K heads, time_step_rank→num V heads, state_size→head dim, conv_kernel). Enable `qwen35_attn_output_gate` — Qwen 3.5/3.6 fuse the attention output gate into a double-wide q_proj, which the graph splits. gguf_archive.cpp: map the qwen35 GGUF tensor names — `attn_qkv`/`attn_gate` → linear_attn.in_proj_qkv/in_proj_z, `ssm_alpha`/`ssm_beta` → in_proj_a/b, `ssm_a`/`ssm_dt.bias`/`ssm_conv1d`/`ssm_norm`/`ssm_out` → A_log/dt_bias/conv1d /norm/out_proj, and the `*_shexp` shared-expert tensors. model.cpp: invert the llama.cpp converter's value transforms at load for a GGUF source (`A_log` stored as `-exp`, `*norm.weight` folded `+1` except `linear_attn.norm`); accept F32/F16 conv1d; and select the multimodal tensor prefix from the archive so a flat text GGUF is not looked up under the `model.language_model.` wrapper. Verified: Qwen3.5-0.8B-Q4_K_M.gguf now boots on portable Metal and completes "The capital of France is" -> "**Paris**. Located in the region of Île-de-France" (coherent). Larger dense (27B) and MoE (35B-A3B) follow-ups: V-head reorder for num_v_heads != num_k_heads, and fused-expert assembly. --- driver/portable/src/gguf_archive.cpp | 24 +++++ driver/portable/src/gguf_archive.hpp | 1 + driver/portable/src/gguf_hparams.cpp | 50 ++++++++++ driver/portable/src/model.cpp | 142 ++++++++++++++++++++++----- driver/portable/src/model.hpp | 10 ++ driver/portable/src/safetensors.hpp | 5 + 6 files changed, 210 insertions(+), 22 deletions(-) diff --git a/driver/portable/src/gguf_archive.cpp b/driver/portable/src/gguf_archive.cpp index c7761c1d7..a3129b73c 100644 --- a/driver/portable/src/gguf_archive.cpp +++ b/driver/portable/src/gguf_archive.cpp @@ -111,6 +111,30 @@ std::string gguf_to_hf_name(const std::string& g) { if (suffix == "ffn_gate_exps.weight") return layer + "mlp.experts.gate_proj.weight"; if (suffix == "ffn_up_exps.weight") return layer + "mlp.experts.up_proj.weight"; if (suffix == "ffn_down_exps.weight") return layer + "mlp.experts.down_proj.weight"; + // Qwen 3.5 / 3.6 shared expert (qwen35moe). `ffn_gate_inp_shexp` is the + // 1-D sigmoid gate over the shared expert (HF `mlp.shared_expert_gate`), + // distinct from the per-token router `ffn_gate_inp`. + if (suffix == "ffn_gate_shexp.weight") return layer + "mlp.shared_expert.gate_proj.weight"; + if (suffix == "ffn_up_shexp.weight") return layer + "mlp.shared_expert.up_proj.weight"; + if (suffix == "ffn_down_shexp.weight") return layer + "mlp.shared_expert.down_proj.weight"; + if (suffix == "ffn_gate_inp_shexp.weight") return layer + "mlp.shared_expert_gate.weight"; + // Qwen 3.5 / 3.6 hybrid gated-delta-rule linear attention. The GGUF + // (llama.cpp qwen35 convention) packs the linear-attn input projection + // as `attn_qkv` (the fused q/k/v of the delta net) + `attn_gate` (the + // output Z gate), and the gated-delta params as `ssm_*`. These names + // only ever appear on the *linear* layers; full-attention layers carry + // the ordinary `attn_q/k/v` handled above. The convert-time value + // transforms (A_log = -exp(raw); folded norm.weight +1) are inverted at + // load in `model.cpp::build_qwen3_5_`, not here. + if (suffix == "attn_qkv.weight") return layer + "linear_attn.in_proj_qkv.weight"; + if (suffix == "attn_gate.weight") return layer + "linear_attn.in_proj_z.weight"; + if (suffix == "ssm_alpha.weight") return layer + "linear_attn.in_proj_a.weight"; + if (suffix == "ssm_beta.weight") return layer + "linear_attn.in_proj_b.weight"; + if (suffix == "ssm_a") return layer + "linear_attn.A_log"; + if (suffix == "ssm_dt.bias") return layer + "linear_attn.dt_bias"; + if (suffix == "ssm_conv1d.weight") return layer + "linear_attn.conv1d.weight"; + if (suffix == "ssm_norm.weight") return layer + "linear_attn.norm.weight"; + if (suffix == "ssm_out.weight") return layer + "linear_attn.out_proj.weight"; return {}; } diff --git a/driver/portable/src/gguf_archive.hpp b/driver/portable/src/gguf_archive.hpp index cb4c000f8..cf6e76d33 100644 --- a/driver/portable/src/gguf_archive.hpp +++ b/driver/portable/src/gguf_archive.hpp @@ -61,6 +61,7 @@ class GGUFArchive : public WeightArchive { const StTensor* find(const std::string& name) const noexcept override; const StTensor& at(const std::string& name) const override; std::size_t num_tensors() const noexcept override { return tensors_.size(); } + bool is_gguf() const noexcept override { return true; } const GgufMeta& meta() const noexcept { return meta_; } diff --git a/driver/portable/src/gguf_hparams.cpp b/driver/portable/src/gguf_hparams.cpp index 49dcc46d5..dd946f479 100644 --- a/driver/portable/src/gguf_hparams.cpp +++ b/driver/portable/src/gguf_hparams.cpp @@ -121,6 +121,56 @@ Hparams parse_gguf_hparams(const GgufMeta& meta) { h.sliding_window = v; } + // ---- Qwen 3.5 / 3.6 hybrid (gated-delta-rule linear attn + GQA) ----- + // The safetensors path fills these from config.json (hf_config.cpp); + // for a GGUF the same facts live under the `qwen35.*` kv namespace. + // Without them `model.cpp::build_qwen3_5_` throws "layer_types missing". + if (h.arch == PieArch::Qwen3_5) { + // Linear-attention dims. The GGUF `ssm.*` keys carry the same + // quantities the HF config exposes as `linear_*`: + // group_count -> num K heads (ssm_n_group) + // time_step_rank -> num V heads (ssm_dt_rank) + // state_size -> K and V head dim (ssm_d_state) + // conv_kernel -> depthwise conv width + // Verified against real blobs by the identity + // ssm.inner_size == num_v_heads * v_head_dim + // (0.8B: 16*128=2048; 27B: 48*128=6144). + h.qwen35_linear_num_k_heads = + static_cast(get_num(meta, a, "ssm.group_count", 0)); + h.qwen35_linear_num_v_heads = + static_cast(get_num(meta, a, "ssm.time_step_rank", 0)); + h.qwen35_linear_k_head_dim = + static_cast(get_num(meta, a, "ssm.state_size", 0)); + h.qwen35_linear_v_head_dim = h.qwen35_linear_k_head_dim; + h.qwen35_linear_conv_kernel = + static_cast(get_num(meta, a, "ssm.conv_kernel", 4)); + + // Per-layer attention type. The GGUF stores only the stride + // `full_attention_interval`; llama.cpp derives the pattern as + // "layer i is full attention iff (i+1) % interval == 0, else + // linear" (every interval-th layer, last of each group). pie + // encodes full='g', linear='l' in `layer_types`. + const std::int32_t interval = static_cast( + get_num(meta, a, "full_attention_interval", 4)); + h.qwen35_full_attn_interval = interval; + if (interval > 0 && h.num_hidden_layers > 0) { + h.layer_types.assign(static_cast(h.num_hidden_layers), 'l'); + for (std::int32_t i = 0; i < h.num_hidden_layers; ++i) { + if (((i + 1) % interval) == 0) { + h.layer_types[static_cast(i)] = 'g'; + } + } + } + + // Qwen 3.5 / 3.6 full-attention layers always carry an output gate + // FUSED into a double-wide `q_proj` (out = 2 * n_heads * head_dim: + // half query, half gate), so there is no separate gate tensor in + // the GGUF. `graph_qwen3_5.cpp` splits it when this flag is set; + // leaving it false reshapes the double-wide Q to the single-width + // head layout and aborts (ggml_nelements mismatch). + h.qwen35_attn_output_gate = true; + } + return h; } diff --git a/driver/portable/src/model.cpp b/driver/portable/src/model.cpp index bdc4bee6e..6aa53b48d 100644 --- a/driver/portable/src/model.cpp +++ b/driver/portable/src/model.cpp @@ -509,10 +509,18 @@ void Model::resolve_tensor_prefix_() { case PieArch::Gemma3n: case PieArch::Qwen3_5: case PieArch::Qwen3VL: - // Gemma 4 / 3n, Qwen 3.5 / 3.6, and Qwen3-VL all ship as multimodal - // ForConditionalGeneration wrappers; the text decoder lives - // under `model.language_model.` (vision tower under `model.visual.`). - tensor_prefix_ = "model.language_model."; + // Gemma 4 / 3n, Qwen 3.5 / 3.6, and Qwen3-VL ship as multimodal + // ForConditionalGeneration wrappers whose safetensors nest the + // text decoder under `model.language_model.`. A GGUF, however, + // is always the flattened text model (llama.cpp drops the + // wrapper, e.g. `token_embd` -> `model.embed_tokens.weight`), so + // only apply the wrapper prefix when the canonical name is + // absent and the wrapped name is present — same archive-driven + // detection the Gemma 3 / Mistral 3 cases use below. + if (archive_->find("model.embed_tokens.weight") == nullptr && + archive_->find("model.language_model.embed_tokens.weight") != nullptr) { + tensor_prefix_ = "model.language_model."; + } break; case PieArch::Gemma3: // Gemma 3 4B+ multimodal (Gemma3ForConditionalGeneration) puts @@ -2222,30 +2230,118 @@ void Model::build_gpt_oss_() { // `[conv_dim, 1, conv_kernel]` BF16; ggml_ssm_conv requires the weight // to be 2D F32 (`[conv_kernel, conv_dim]`). Squeeze + convert at load // time using a synth tensor. +// Read a small source weight (BF16 safetensors or BF16/F16/F32 GGUF) as a +// flat F32 vector. Used by the Qwen 3.5 synth helpers below, which need the +// raw float values to invert convert-time transforms. The element count is +// derived from the source dtype so it works regardless of how the tensor was +// stored. Throws on an unexpected (quantized) source. +static std::vector read_small_weight_as_f32_(const StTensor& src, + const std::string& hf_name) { + // GGUF tensors carry their ggml type in `ggml_type_override`; safetensors + // leaves it -1 and uses `dtype`. + const bool is_gguf = src.ggml_type_override >= 0; + enum { TF32 = 0, TF16 = 1, TBF16 = 24 }; // ggml_type values + std::size_t elem_bytes; + int kind; // 0=f32, 1=f16, 2=bf16 + if (is_gguf) { + switch (src.ggml_type_override) { + case TF32: elem_bytes = 4; kind = 0; break; + case TF16: elem_bytes = 2; kind = 1; break; + case TBF16: elem_bytes = 2; kind = 2; break; + default: + throw std::runtime_error( + "model qwen3_5: expected an unquantized small weight for " + + hf_name); + } + } else if (src.dtype == StDtype::BF16) { + elem_bytes = 2; kind = 2; + } else if (src.dtype == StDtype::F32) { + elem_bytes = 4; kind = 0; + } else { + throw std::runtime_error( + "model qwen3_5: unsupported small-weight dtype for " + hf_name); + } + const std::size_t n = src.nbytes / elem_bytes; + std::vector out(n); + if (kind == 0) { + std::memcpy(out.data(), src.data, n * sizeof(float)); + } else if (kind == 2) { + const auto* p = reinterpret_cast(src.data); + for (std::size_t i = 0; i < n; ++i) out[i] = bf16_to_f32(p[i]); + } else { // f16 + const auto* p = reinterpret_cast(src.data); + for (std::size_t i = 0; i < n; ++i) out[i] = ggml_fp16_to_fp32(p[i]); + } + return out; +} + ggml_tensor* Model::declare_qwen3_5_conv1d_(const std::string& hf_name, std::int32_t conv_dim, std::int32_t conv_kernel) { const auto& src = archive_->at(hf_name); - if (src.dtype != StDtype::BF16) { - throw std::runtime_error( - "model qwen3_5: conv1d weight must be BF16 for " + hf_name); - } + // HF/safetensors ships BF16; the GGUF qwen35 converter squeezes the + // depthwise conv to a 2-D F32 (sometimes F16) tensor. Accept both. + const std::vector vals = read_small_weight_as_f32_(src, hf_name); + auto* t = ggml_new_tensor_2d(ctx_, GGML_TYPE_F32, conv_kernel, conv_dim); ggml_set_name(t, hf_name.c_str()); - const std::size_t n_elems = src.nbytes / sizeof(std::uint16_t); - if (static_cast(n_elems) != conv_dim * conv_kernel) { + if (static_cast(vals.size()) != conv_dim * conv_kernel) { throw std::runtime_error( "model qwen3_5: conv1d weight size mismatch for " + hf_name); } SynthTensor s; s.tensor = t; - s.data.resize(n_elems * sizeof(float)); - auto* dst_f32 = reinterpret_cast(s.data.data()); - const auto* src_bf16 = reinterpret_cast(src.data); - for (std::size_t i = 0; i < n_elems; ++i) { - dst_f32[i] = bf16_to_f32(src_bf16[i]); + s.data.resize(vals.size() * sizeof(float)); + std::memcpy(s.data.data(), vals.data(), vals.size() * sizeof(float)); + synth_.push_back(std::move(s)); + return t; +} + +// `A_log` inverse: the GGUF stores `-exp(A_log)`; the graph wants the raw +// `A_log` (it applies `exp`). For a safetensors source the value is already +// raw, so pass through to `declare_`. +ggml_tensor* Model::declare_qwen3_5_a_log_(const std::string& hf_name) { + const auto& src = archive_->at(hf_name); + if (!archive_->is_gguf()) return declare_(hf_name); + + std::vector vals = read_small_weight_as_f32_(src, hf_name); + for (float& v : vals) v = std::log(-v); // log(-(-exp(a))) == a + + std::int64_t ne[4] = {1, 1, 1, 1}; + for (std::size_t i = 0; i < src.shape.size(); ++i) { + ne[src.shape.size() - 1 - i] = src.shape[i]; } + auto* t = ggml_new_tensor_4d(ctx_, GGML_TYPE_F32, ne[0], ne[1], ne[2], ne[3]); + ggml_set_name(t, hf_name.c_str()); + SynthTensor s; + s.tensor = t; + s.data.resize(vals.size() * sizeof(float)); + std::memcpy(s.data.data(), vals.data(), vals.size() * sizeof(float)); + synth_.push_back(std::move(s)); + return t; +} + +// Folded-norm inverse: the GGUF stores `1 + w` for every `*norm.weight` +// except `linear_attn.norm`; the graph applies `plus_one=true` and so wants +// the raw `w`. For a safetensors source the value is already raw. +ggml_tensor* Model::declare_qwen3_5_folded_norm_(const std::string& hf_name) { + const auto& src = archive_->at(hf_name); + if (!archive_->is_gguf()) return declare_(hf_name); + + std::vector vals = read_small_weight_as_f32_(src, hf_name); + for (float& v : vals) v -= 1.0f; + + std::int64_t ne[4] = {1, 1, 1, 1}; + for (std::size_t i = 0; i < src.shape.size(); ++i) { + ne[src.shape.size() - 1 - i] = src.shape[i]; + } + auto* t = ggml_new_tensor_4d(ctx_, GGML_TYPE_F32, ne[0], ne[1], ne[2], ne[3]); + ggml_set_name(t, hf_name.c_str()); + SynthTensor s; + s.tensor = t; + s.data.resize(vals.size() * sizeof(float)); + std::memcpy(s.data.data(), vals.data(), vals.size() * sizeof(float)); synth_.push_back(std::move(s)); return t; } @@ -2266,7 +2362,7 @@ void Model::build_qwen3_5_() { // ── top level ── (multimodal-wrapped via tensor_prefix_) weights_.tok_embd = declare_(tname_("model.embed_tokens.weight")); - weights_.output_norm = declare_(tname_("model.norm.weight")); + weights_.output_norm = declare_qwen3_5_folded_norm_(tname_("model.norm.weight")); if (!h.tie_word_embeddings) { weights_.output_head = declare_("lm_head.weight"); } @@ -2286,9 +2382,9 @@ void Model::build_qwen3_5_() { const std::string p = tname_(layer_path(i)); auto& L = weights_.layers[i]; - // RMSNorms (same names on both layer types). - L.attn_norm = declare_(p + "input_layernorm.weight"); - L.ffn_norm = declare_(p + "post_attention_layernorm.weight"); + // RMSNorms (same names on both layer types). GGUF folds +1. + L.attn_norm = declare_qwen3_5_folded_norm_(p + "input_layernorm.weight"); + L.ffn_norm = declare_qwen3_5_folded_norm_(p + "post_attention_layernorm.weight"); // FFN: dense SwiGLU on Qwen 3.5; fused-stacked MoE + shared expert // on Qwen 3.6 (qwen3_5_moe). The MoE layout matches HF's @@ -2313,8 +2409,10 @@ void Model::build_qwen3_5_() { L.lin_in_proj_z = declare_(p + "linear_attn.in_proj_z.weight"); L.lin_in_proj_a = declare_(p + "linear_attn.in_proj_a.weight"); L.lin_in_proj_b = declare_(p + "linear_attn.in_proj_b.weight"); - L.lin_A_log = declare_(p + "linear_attn.A_log"); + L.lin_A_log = declare_qwen3_5_a_log_(p + "linear_attn.A_log"); L.lin_dt_bias = declare_(p + "linear_attn.dt_bias"); + // linear_attn.norm is the ONE norm the GGUF converter does not + // fold +1 into, so it loads raw (the graph uses plus_one=false). L.lin_norm = declare_(p + "linear_attn.norm.weight"); L.lin_out_proj = declare_(p + "linear_attn.out_proj.weight"); L.lin_conv1d = declare_qwen3_5_conv1d_( @@ -2326,8 +2424,8 @@ void Model::build_qwen3_5_() { L.k_proj = declare_(p + "self_attn.k_proj.weight"); L.v_proj = declare_(p + "self_attn.v_proj.weight"); L.o_proj = declare_(p + "self_attn.o_proj.weight"); - L.q_norm = declare_(p + "self_attn.q_norm.weight"); - L.k_norm = declare_(p + "self_attn.k_norm.weight"); + L.q_norm = declare_qwen3_5_folded_norm_(p + "self_attn.q_norm.weight"); + L.k_norm = declare_qwen3_5_folded_norm_(p + "self_attn.k_norm.weight"); } else { throw std::runtime_error( "model qwen3_5: unexpected layer_type '" + diff --git a/driver/portable/src/model.hpp b/driver/portable/src/model.hpp index 74a7e2fe0..aaad3356f 100644 --- a/driver/portable/src/model.hpp +++ b/driver/portable/src/model.hpp @@ -412,6 +412,16 @@ class Model { std::int32_t conv_dim, std::int32_t conv_kernel); + // Qwen 3.5 / 3.6 GGUF value-transform inverses. A GGUF produced by + // llama.cpp's qwen35 converter pre-applies two transforms our + // safetensors-shaped graph does not expect: `A_log` is stored as + // `-exp(A_log)`, and every `*norm.weight` except `linear_attn.norm` + // is folded `+1`. These synth the inverse (raw `A_log = log(-x)`; + // raw norm `w = x - 1`) at load so the shared graph is unchanged. + // No-op passthrough to `declare_` for a safetensors source. + ggml_tensor* declare_qwen3_5_a_log_(const std::string& hf_name); + ggml_tensor* declare_qwen3_5_folded_norm_(const std::string& hf_name); + // Load tok_embd / output_norm / (optional) output_head and resize the // per-layer vector. Common to all archs. void load_top_level_(); diff --git a/driver/portable/src/safetensors.hpp b/driver/portable/src/safetensors.hpp index 1d8bbc807..89a884322 100644 --- a/driver/portable/src/safetensors.hpp +++ b/driver/portable/src/safetensors.hpp @@ -111,6 +111,11 @@ class WeightArchive { virtual const StTensor* find(const std::string& name) const noexcept = 0; virtual const StTensor& at(const std::string& name) const = 0; virtual std::size_t num_tensors() const noexcept = 0; + // True when the weights came from a GGUF file. Per-arch loaders use + // this to invert convert-time value transforms (e.g. the Qwen 3.5 + // A_log = -exp and folded-norm +1) that a safetensors source does not + // carry. Defaults false; only `GGUFArchive` overrides it. + virtual bool is_gguf() const noexcept { return false; } }; // Top-level safetensors archive: handles single-file or sharded models. From b57312014acae0f980784031a610948da22470a4 Mon Sep 17 00:00:00 2001 From: Seung-seob Lee Date: Tue, 16 Jun 2026 21:45:31 -0400 Subject: [PATCH 2/2] portable: qwen35 GGUF support for unequal-head and MoE checkpoints Extends the qwen35/qwen35moe GGUF loader past the small dense smoke case: - GQA V-head order: the converter permutes the linear-attention V heads grouped->tiled when num_v_heads != num_k_heads (so ggml's tiled broadcast aligns), while HF safetensors keep grouped order. Carry a `qwen35_linear_v_tiled` hparam (set for a GGUF source) and have the graph expand Q/K with a matching tiled `ggml_repeat` instead of the grouped `repeat_interleave`. The equal-head case is unaffected. - MoE: read `expert_shared_feed_forward_length` into `shared_expert_intermediate_size`, and load the experts from the GGUF's three SEPARATE stacked tensors (`ffn_gate_exps` / `ffn_up_exps` / `ffn_down_exps`) plus the shared-expert tensors directly, rather than splitting a fused `mlp.experts.gate_up_proj` (which only the safetensors layout provides). Verified on portable Metal: Qwen3.6-27B-Q4_K_M (dense, 64 layers) and Qwen3.6-35B-A3B-UD-Q4_K_M (MoE, 256 experts + shared expert) both boot and complete a prompt ("The capital of France is" -> "Paris"; coherent MoE reasoning output). --- driver/portable/src/gguf_hparams.cpp | 9 +++++++++ driver/portable/src/graph_qwen3_5.cpp | 25 +++++++++++++++++-------- driver/portable/src/hf_config.hpp | 7 +++++++ driver/portable/src/model.cpp | 17 +++++++++++++++++ 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/driver/portable/src/gguf_hparams.cpp b/driver/portable/src/gguf_hparams.cpp index dd946f479..eb1b53b47 100644 --- a/driver/portable/src/gguf_hparams.cpp +++ b/driver/portable/src/gguf_hparams.cpp @@ -144,6 +144,15 @@ Hparams parse_gguf_hparams(const GgufMeta& meta) { h.qwen35_linear_v_head_dim = h.qwen35_linear_k_head_dim; h.qwen35_linear_conv_kernel = static_cast(get_num(meta, a, "ssm.conv_kernel", 4)); + // A GGUF always carries the converter's tiled V-head order; the + // graph must expand Q/K with a tiled broadcast to match. + h.qwen35_linear_v_tiled = true; + + // qwen35moe shared expert (always-active dense path alongside the + // routed experts). The generic MoE block above already read + // expert_count / expert_used_count / expert_feed_forward_length. + h.shared_expert_intermediate_size = static_cast( + get_num(meta, a, "expert_shared_feed_forward_length", 0)); // Per-layer attention type. The GGUF stores only the stride // `full_attention_interval`; llama.cpp derives the pattern as diff --git a/driver/portable/src/graph_qwen3_5.cpp b/driver/portable/src/graph_qwen3_5.cpp index cdb0e0877..5eabecd5b 100644 --- a/driver/portable/src/graph_qwen3_5.cpp +++ b/driver/portable/src/graph_qwen3_5.cpp @@ -107,18 +107,27 @@ ggml_tensor* build_qwen3_5_linear_layer( auto* k = ggml_reshape_4d(ctx, k_flat, S, n_kh, n_tokens, n_seqs); auto* v = ggml_reshape_4d(ctx, v_flat, S, n_vh, n_tokens, n_seqs); - // ---- 3a. GQA: repeat_interleave Q/K along the head axis to match V. - // Q' [s, k*f+i, t, b] = Q[s, k, t, b] for all i in [0, f). - // Implemented as reshape→repeat_4d (tile along inserted dim)→reshape. + // ---- 3a. GQA: expand Q/K along the head axis to match V's n_vh heads. + // The expansion order must match how V's heads are laid out: + // · grouped (HF safetensors): Q'[s, k*f+i] = Q[s, k] — repeat_interleave + // · tiled (GGUF converter): Q'[s, k + n_kh*i] = Q[s, k] — ggml_repeat + // Both via reshape→repeat_4d→reshape; only the inserted-dim position + // differs (before vs after the head axis). if (gqa_f > 1) { - auto interleave = [&](ggml_tensor* x) { - x = ggml_reshape_4d(ctx, x, S, 1, n_kh, n_tokens * n_seqs); - x = ggml_repeat_4d(ctx, x, S, gqa_f, n_kh, n_tokens * n_seqs); + const bool tiled = h.qwen35_linear_v_tiled; + auto expand = [&](ggml_tensor* x) { + if (tiled) { + x = ggml_reshape_4d(ctx, x, S, n_kh, 1, n_tokens * n_seqs); + x = ggml_repeat_4d(ctx, x, S, n_kh, gqa_f, n_tokens * n_seqs); + } else { + x = ggml_reshape_4d(ctx, x, S, 1, n_kh, n_tokens * n_seqs); + x = ggml_repeat_4d(ctx, x, S, gqa_f, n_kh, n_tokens * n_seqs); + } x = ggml_cont(ctx, x); return ggml_reshape_4d(ctx, x, S, H, n_tokens, n_seqs); }; - q = interleave(q); - k = interleave(k); + q = expand(q); + k = expand(k); } // ---- 4. L2-norm. The fused op applies the 1/sqrt(S) Q-scale itself diff --git a/driver/portable/src/hf_config.hpp b/driver/portable/src/hf_config.hpp index 16d818d42..933db91cd 100644 --- a/driver/portable/src/hf_config.hpp +++ b/driver/portable/src/hf_config.hpp @@ -270,6 +270,13 @@ struct Hparams { // mrope + output gate. Vision tower + multi-token-prediction head // are present in checkpoints but ignored by the driver. bool qwen35_attn_output_gate = false; + // GGUF stores the linear-attention V heads in *tiled* order (the + // llama.cpp qwen35 converter permutes grouped->tiled when + // num_v_heads != num_k_heads so ggml's tiled broadcast aligns); HF + // safetensors keep the original grouped order. The graph expands Q/K + // to match V using a tiled `ggml_repeat` when set, else a grouped + // `repeat_interleave`. + bool qwen35_linear_v_tiled = false; std::int32_t qwen35_full_attn_interval = 0; std::int32_t qwen35_linear_num_k_heads = 0; std::int32_t qwen35_linear_num_v_heads = 0; diff --git a/driver/portable/src/model.cpp b/driver/portable/src/model.cpp index 6aa53b48d..a48baa55d 100644 --- a/driver/portable/src/model.cpp +++ b/driver/portable/src/model.cpp @@ -961,6 +961,23 @@ void Model::load_qwen3_5_moe_layer_(std::int32_t i, const std::string& p) { // Router: HF naming `mlp.gate.weight`, shape [num_experts, hidden]. L.moe_router = declare_(p + "mlp.gate.weight"); + // GGUF path: llama.cpp stores the experts as three SEPARATE stacked + // tensors (`ffn_gate_exps` / `ffn_up_exps` / `ffn_down_exps`, mapped to + // `mlp.experts.{gate,up,down}_proj`) already in ggml's mul_mat_id + // layout, so declare each directly — no fused gate_up split. + if (archive_->is_gguf()) { + L.moe_gate_exps = declare_(p + "mlp.experts.gate_proj.weight"); + L.moe_up_exps = declare_(p + "mlp.experts.up_proj.weight"); + L.moe_down_exps = declare_(p + "mlp.experts.down_proj.weight"); + if (h.shared_expert_intermediate_size > 0) { + L.moe_shared_gate_proj = declare_(p + "mlp.shared_expert.gate_proj.weight"); + L.moe_shared_up_proj = declare_(p + "mlp.shared_expert.up_proj.weight"); + L.moe_shared_down_proj = declare_(p + "mlp.shared_expert.down_proj.weight"); + L.moe_shared_gate = declare_(p + "mlp.shared_expert_gate.weight"); + } + return; + } + // Fused experts. Source `mlp.experts.gate_up_proj` is one 3D safetensor // [n_exp, 2*ff, hidden]; we split into separate stacked gate / up // tensors at load time so the existing build_moe_ffn helper applies.