Upload folder using huggingface_hub

README.md

@@ -4,43 +4,25 @@ tags:
 - mergekit
 - lazymergekit
 - SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
 base_model:
 - SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
-- SciPhi/SciPhi-Mistral-7B-32k
 ---
 
 # SciPhi-Mistral-7B-32k-sliced
 
 SciPhi-Mistral-7B-32k-sliced is a merge of the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
 * [SciPhi/SciPhi-Mistral-7B-32k](https://huggingface.co/SciPhi/SciPhi-Mistral-7B-32k)
-* [SciPhi/SciPhi-Mistral-7B-32k](https://huggingface.co/SciPhi/SciPhi-Mistral-7B-32k)
-* [SciPhi/SciPhi-Mistral-7B-32k](https://huggingface.co/SciPhi/SciPhi-Mistral-7B-32k)
-* [SciPhi/SciPhi-Mistral-7B-32k](https://huggingface.co/SciPhi/SciPhi-Mistral-7B-32k)
-* [SciPhi/SciPhi-Mistral-7B-32k](https://huggingface.co/SciPhi/SciPhi-Mistral-7B-32k)
-* [SciPhi/SciPhi-Mistral-7B-32k](https://huggingface.co/SciPhi/SciPhi-Mistral-7B-32k)
-* [SciPhi/SciPhi-Mistral-7B-32k](https://huggingface.co/SciPhi/SciPhi-Mistral-7B-32k)
-* [SciPhi/SciPhi-Mistral-7B-32k](https://huggingface.co/SciPhi/SciPhi-Mistral-7B-32k)
-* [SciPhi/SciPhi-Mistral-7B-32k](https://huggingface.co/SciPhi/SciPhi-Mistral-7B-32k)
 
 ## 🧩 Configuration
 
 ```yaml
 slices:
+  - sources:
+      - model: SciPhi/SciPhi-Mistral-7B-32k
+        layer_range: [0, 0]
+  - sources:
+      - model: SciPhi/SciPhi-Mistral-7B-32k
+        layer_range: [1, 1]
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
         layer_range: [3, 3]
@@ -68,12 +50,21 @@ slices:
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
         layer_range: [23, 23]
-
-
-merge_method: passthrough
-tokenizer_source: union
-
+  - sources:
+      - model: SciPhi/SciPhi-Mistral-7B-32k
+        layer_range: [32, 32]
+merge_method: slerp
+base_model: Locutusque/TinyMistral-248M-v2.5-Instruct
+parameters:
+  t:
+  - filter: self_attn
+    value: [0, 0.5, 0.3, 0.7, 1]
+  - filter: mlp
+    value: [1, 0.5, 0.7, 0.3, 0]
+  - value: 0.3
 dtype: float16
+tokenizer_source: base
+
 ```
 
 ## 💻 Usage

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "SciPhi/SciPhi-Mistral-7B-32k",
+  "_name_or_path": "Locutusque/TinyMistral-248M-v2.5-Instruct",
   "architectures": [
     "MistralForCausalLM"
   ],
@@ -7,20 +7,20 @@
   "bos_token_id": 1,
   "eos_token_id": 2,
   "hidden_act": "silu",
-  "hidden_size": 4096,
+  "hidden_size": 1024,
   "initializer_range": 0.02,
-  "intermediate_size": 14336,
+  "intermediate_size": 4096,
   "max_position_embeddings": 32768,
   "model_type": "mistral",
   "num_attention_heads": 32,
   "num_hidden_layers": 0,
   "num_key_value_heads": 8,
-  "rms_norm_eps": 1e-05,
+  "rms_norm_eps": 1e-06,
   "rope_theta": 10000.0,
-  "sliding_window": 4096,
+  "sliding_window": 32,
   "tie_word_embeddings": false,
   "torch_dtype": "float16",
   "transformers_version": "4.37.2",
-  "use_cache": false,
-  "vocab_size": 32000
+  "use_cache": true,
+  "vocab_size": 32005
 }

mergekit_config.yml

@@ -1,5 +1,11 @@
 
 slices:
+  - sources:
+      - model: SciPhi/SciPhi-Mistral-7B-32k
+        layer_range: [0, 0]
+  - sources:
+      - model: SciPhi/SciPhi-Mistral-7B-32k
+        layer_range: [1, 1]
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
         layer_range: [3, 3]
@@ -27,9 +33,18 @@ slices:
   - sources:
       - model: SciPhi/SciPhi-Mistral-7B-32k
         layer_range: [23, 23]
-
-
-merge_method: passthrough
-tokenizer_source: union
-
+  - sources:
+      - model: SciPhi/SciPhi-Mistral-7B-32k
+        layer_range: [32, 32]
+merge_method: slerp
+base_model: Locutusque/TinyMistral-248M-v2.5-Instruct
+parameters:
+  t:
+  - filter: self_attn
+    value: [0, 0.5, 0.3, 0.7, 1]
+  - filter: mlp
+    value: [1, 0.5, 0.7, 0.3, 0]
+  - value: 0.3
 dtype: float16
+tokenizer_source: base
+

special_tokens_map.json

@@ -1,18 +1,20 @@
 {
-  "additional_special_tokens": [
-    "<unk>",
-    "<s>",
-    "</s>"
-  ],
   "bos_token": {
-    "content": "<s>",
+    "content": "<|bos|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "</s>",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,7 +1,21 @@
 {
   "version": "1.0",
-  "truncation": null,
-  "padding": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
+  "padding": {
+    "strategy": {
+      "Fixed": 512
+    },
+    "direction": "Left",
+    "pad_to_multiple_of": null,
+    "pad_id": 32001,
+    "pad_type_id": 0,
+    "pad_token": "<|endoftext|>"
+  },
   "added_tokens": [
     {
       "id": 0,
@@ -10,7 +24,7 @@
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 1,
@@ -19,7 +33,7 @@
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 2,
@@ -28,7 +42,52 @@
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
+    },
+    {
+      "id": 32000,
+      "content": "<|bos|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32001,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32002,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32003,
+      "content": "<|ASSISTANT|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32004,
+      "content": "<|USER|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": {
@@ -53,7 +112,7 @@
     "single": [
       {
         "SpecialToken": {
-          "id": "<s>",
+          "id": "<|bos|>",
           "type_id": 0
         }
       },
@@ -67,7 +126,7 @@
     "pair": [
       {
         "SpecialToken": {
-          "id": "<s>",
+          "id": "<|bos|>",
           "type_id": 0
         }
       },
@@ -79,7 +138,7 @@
       },
       {
         "SpecialToken": {
-          "id": "<s>",
+          "id": "<|bos|>",
           "type_id": 1
         }
       },
@@ -91,13 +150,13 @@
       }
     ],
     "special_tokens": {
-      "<s>": {
-        "id": "<s>",
+      "<|bos|>": {
+        "id": "<|bos|>",
         "ids": [
-          1
+          32000
         ],
         "tokens": [
-          "<s>"
+          "<|bos|>"
         ]
       }
     }

tokenizer_config.json

@@ -8,7 +8,7 @@
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "1": {
       "content": "<s>",
@@ -16,7 +16,7 @@
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "2": {
       "content": "</s>",
@@ -24,23 +24,66 @@
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
+    },
+    "32000": {
+      "content": "<|bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|ASSISTANT|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|USER|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
-  "additional_special_tokens": [
-    "<unk>",
-    "<s>",
-    "</s>"
-  ],
-  "bos_token": "<s>",
+  "additional_special_tokens": [],
+  "bos_token": "<|bos|>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
+  "eos_token": "<|endoftext|>",
   "legacy": true,
+  "max_length": 1536,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": null,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
+  "stride": 0,
   "tokenizer_class": "LlamaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>",
   "use_default_system_prompt": true
 }