make the default model smaller: width 512, 4 query heads

proger · Mar 11, 2024 · ee62f7d · ee62f7d
1 parent 4aec172
commit ee62f7d
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/hippogriff.py b/hippogriff.py
@@ -13,9 +13,9 @@
 class GriffinConfig:
     vocab_size: int = 256
     num_layers: int = 1
-    dim: int = 1024
+    dim: int = 512
     smqa_head_dim: int = 128
-    smqa_q_heads: int = 8
+    smqa_q_heads: int = 4
     smqa_kv_heads: int = 1
     smqa_window_size: int = 512
     hawk_expansion_factor: float = 1.5