Upload folder using huggingface_hub

README.md

@@ -19,52 +19,31 @@ SciPhi-Mistral-7B-32k-sliced is a merge of the following models using [LazyMerge
 slices:
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [0, 0]
+        layer_range: [0, 3]
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [1, 1]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [3, 3]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [5, 5]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [6, 6]
+        layer_range: [5, 7]
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
         layer_range: [10, 10]
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [17, 17]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [18, 18]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [19, 19]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [20, 20]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [23, 23]
+        layer_range: [17, 23]
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [32, 32]
+        layer_range: [31, 32]
   - sources:
       - model: Locutusque/TinyMistral-248M-v2.5
-        layer_range: [0, 0]
+        layer_range: [0, 1]
   - sources:
       - model: Locutusque/TinyMistral-248M-v2.5
-        layer_range: [11, 11]
+        layer_range: [11, 12]
   - sources:
       - model: Locutusque/TinyMistral-248M-v2.5-Instruct
-        layer_range: [0, 0]
+        layer_range: [0, 1]
   - sources:
       - model: Locutusque/TinyMistral-248M-v2.5-Instruct
-        layer_range: [11, 11]
+        layer_range: [11, 12]
 merge_method: slerp
 base_model: Locutusque/TinyMistral-248M-v2.5-Instruct
 parameters:

config.json

@@ -13,7 +13,7 @@
   "max_position_embeddings": 32768,
   "model_type": "mistral",
   "num_attention_heads": 32,
-  "num_hidden_layers": 0,
+  "num_hidden_layers": 16,
   "num_key_value_heads": 8,
   "rms_norm_eps": 1e-06,
   "rope_theta": 10000.0,

mergekit_config.yml

@@ -2,52 +2,31 @@
 slices:
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [0, 0]
+        layer_range: [0, 3]
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [1, 1]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [3, 3]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [5, 5]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [6, 6]
+        layer_range: [5, 7]
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
         layer_range: [10, 10]
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [17, 17]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [18, 18]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [19, 19]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [20, 20]
-  - sources:
-      - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [23, 23]
+        layer_range: [17, 23]
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
-        layer_range: [32, 32]
+        layer_range: [31, 32]
   - sources:
       - model: Locutusque/TinyMistral-248M-v2.5
-        layer_range: [0, 0]
+        layer_range: [0, 1]
   - sources:
       - model: Locutusque/TinyMistral-248M-v2.5
-        layer_range: [11, 11]
+        layer_range: [11, 12]
   - sources:
       - model: Locutusque/TinyMistral-248M-v2.5-Instruct
-        layer_range: [0, 0]
+        layer_range: [0, 1]
   - sources:
       - model: Locutusque/TinyMistral-248M-v2.5-Instruct
-        layer_range: [11, 11]
+        layer_range: [11, 12]
 merge_method: slerp
 base_model: Locutusque/TinyMistral-248M-v2.5-Instruct
 parameters:

model-00001-of-00001.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db00d61113ef1c6db26043cdc2f6be07fc9a0b9f9c3e3a3d77cd050d5b21acf2
-size 327692600
+oid sha256:b25afb4975084363df26d6b6cf49fa6ab17ea46996f7acad6965a8a45929d388
+size 1636367680

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39f4175ec1c3299d5ba3755fc0f37c36899400ca8a6db160d0a89880a9a60582
+size 1932103880

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e9114872344e623ef09e1fcbd0875739938205736893dc19d5771c4353478f1
+size 1889587008

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:104ea58074879641867c4c0875dca16c85f41153450b9aac7a65c4f5b66a3dca
+size 1862357376

model.safetensors.index.json

@@ -1 +1 @@
-{"metadata": {"mergekit_version": "0.0.4.1"}, "weight_map": {"lm_head.weight": "model-00001-of-00001.safetensors", "model.norm.weight": "model-00001-of-00001.safetensors", "model.embed_tokens.weight": "model-00001-of-00001.safetensors"}}
+{"metadata": {"mergekit_version": "0.0.4.1"}, "weight_map": {"lm_head.weight": "model-00001-of-00003.safetensors", "model.norm.weight": "model-00001-of-00003.safetensors", "model.layers.15.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.15.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.14.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.14.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.13.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.12.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", "model.layers.0.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.0.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.0.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.0.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.0.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.0.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.0.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.embed_tokens.weight": "model-00002-of-00003.safetensors", "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors", "model.layers.7.mlp.down_proj.weight": "model-00002-of-00003.safetensors", "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", "model.layers.7.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.7.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.6.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.6.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.5.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.5.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.11.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.11.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", "model.layers.11.input_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.10.mlp.down_proj.weight": "model-00003-of-00003.safetensors", "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", "model.layers.10.mlp.up_proj.weight": "model-00003-of-00003.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", "model.layers.10.input_layernorm.weight": "model-00003-of-00003.safetensors"}}