From 10cfc245e49d0a2fe26d6e4247bf486a6a948704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E5=AE=87=E6=89=AC?= Date: Thu, 27 Jun 2024 08:58:17 +0800 Subject: [PATCH] =?UTF-8?q?moe=E6=A8=A1=E5=9E=8B=E8=A1=A5=E5=85=85weightNa?= =?UTF-8?q?me?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/models/moe.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/models/moe.cpp b/src/models/moe.cpp index 1775e135..c8f25b17 100644 --- a/src/models/moe.cpp +++ b/src/models/moe.cpp @@ -33,6 +33,13 @@ namespace fastllm { rotary_dim = 128; weight.embeddingNames.insert("model.embed_tokens.weight"); + weight.linearNames = { + "lm_head.weight", "model.layers.*.down_proj.weight", "model.layers.*.up_proj.weight", + "model.layers.*.gate_proj.weight", "model.layers.*.gate_proj.weight", "model.layers.*.gateup_proj.weight", + "model.layers.*.self_attn.o_proj.weight", "model.layers.*.self_attn.q_proj.weight", "model.layers.*.self_attn.k_proj.weight", + "model.layers.*.self_attn.v_proj.weight", "model.layers.*.self_attn.mergeqkv.weight", "model.layers.*.self_attn.W_pack.weight", + "model.layers.*.mlp.*.weight" + }; } void MoeModel::InitParams() {