Merged (Healed) Model from CPU Slicing+Stitch+LoRA

config.json

@@ -15,9 +15,9 @@
   "intermediate_size": 16384,
   "max_position_embeddings": 8192,
   "model_type": "gemma",
-  "num_attention_heads": 10,
+  "num_attention_heads": 8,
   "num_hidden_layers": 8,
-  "num_key_value_heads": 10,
+  "num_key_value_heads": 8,
   "pad_token_id": 0,
   "rms_norm_eps": 1e-06,
   "rope_theta": 10000.0,

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:350b34ea799fbfb5de7c983810f34e20970374bbd88b6a5dc286c4499cc2f394
-size 4882257768
+oid sha256:1d90225c2487447c4472adaed2fed2968f063b1646f8ed60cdf7ca1dc7483736
+size 4982938256

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b5ce83bba37f36d9d38669a6ec55bdce5175474163efe190f97003f64b440a3
-size 1107356096
+oid sha256:6cc8bc234e3df7cd20f4d934b8d155eab13588f2d53a2c49f2e9977ba7b1e9b1
+size 872457880

model.safetensors.index.json

@@ -1,7 +1,7 @@
 {
   "metadata": {
-    "total_parameters": 1497401344,
-    "total_size": 5989605376
+    "total_parameters": 1463846912,
+    "total_size": 5855387648
   },
   "weight_map": {
     "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
@@ -50,11 +50,11 @@
     "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
@@ -64,10 +64,10 @@
     "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.6.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.7.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",