frost19k commited on
Commit
42e296b
1 Parent(s): d365a03

Upload folder using huggingface_hub

Browse files
README.md ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ tags:
4
+ - moe
5
+ - frankenmoe
6
+ - merge
7
+ - mergekit
8
+ - lazymergekit
9
+ - cognitivecomputations/dolphin-2.8-mistral-7b-v02
10
+ - OpenPipe/mistral-ft-optimized-1227
11
+ base_model:
12
+ - cognitivecomputations/dolphin-2.8-mistral-7b-v02
13
+ - OpenPipe/mistral-ft-optimized-1227
14
+ ---
15
+
16
+ # dolphin-2.8-mistral-v02-2x7b
17
+
18
+ dolphin-2.8-mistral-v02-2x7b is a Mixture of Experts (MoE) made with the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
19
+ * [cognitivecomputations/dolphin-2.8-mistral-7b-v02](https://huggingface.co/cognitivecomputations/dolphin-2.8-mistral-7b-v02)
20
+ * [OpenPipe/mistral-ft-optimized-1227](https://huggingface.co/OpenPipe/mistral-ft-optimized-1227)
21
+
22
+ ## 🧩 Configuration
23
+
24
+ ```yaml
25
+ base_model: cognitivecomputations/dolphin-2.8-mistral-7b-v02
26
+ experts:
27
+ - source_model: cognitivecomputations/dolphin-2.8-mistral-7b-v02
28
+ positive_prompts:
29
+ - "chat"
30
+ - "assistant"
31
+ - "tell me"
32
+ - "explain"
33
+ - "I want"
34
+ - source_model: OpenPipe/mistral-ft-optimized-1227
35
+ positive_prompts:
36
+ - "code"
37
+ - "python"
38
+ - "javascript"
39
+ - "programming"
40
+ - "algorithm"
41
+ ```
42
+
43
+ ## 💻 Usage
44
+
45
+ ```python
46
+ !pip install -qU transformers bitsandbytes accelerate
47
+
48
+ from transformers import AutoTokenizer
49
+ import transformers
50
+ import torch
51
+
52
+ model = "frost19k/dolphin-2.8-mistral-v02-2x7b"
53
+
54
+ tokenizer = AutoTokenizer.from_pretrained(model)
55
+ pipeline = transformers.pipeline(
56
+ "text-generation",
57
+ model=model,
58
+ model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
59
+ )
60
+
61
+ messages = [{"role": "user", "content": "Explain what a Mixture of Experts is in less than 100 words."}]
62
+ prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
63
+ outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
64
+ print(outputs[0]["generated_text"])
65
+ ```
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "<|im_end|>": 32000,
3
+ "<|im_start|>": 32001
4
+ }
config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "cognitivecomputations/dolphin-2.8-mistral-7b-v02",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 32000,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 4096,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 14336,
13
+ "max_position_embeddings": 32768,
14
+ "model_type": "mixtral",
15
+ "num_attention_heads": 32,
16
+ "num_experts_per_tok": 2,
17
+ "num_hidden_layers": 32,
18
+ "num_key_value_heads": 8,
19
+ "num_local_experts": 2,
20
+ "output_router_logits": false,
21
+ "rms_norm_eps": 1e-05,
22
+ "rope_theta": 1000000.0,
23
+ "router_aux_loss_coef": 0.001,
24
+ "router_jitter_noise": 0.0,
25
+ "sliding_window": null,
26
+ "tie_word_embeddings": false,
27
+ "torch_dtype": "bfloat16",
28
+ "transformers_version": "4.40.2",
29
+ "use_cache": false,
30
+ "vocab_size": 32002
31
+ }
mergekit_moe_config.yml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ base_model: cognitivecomputations/dolphin-2.8-mistral-7b-v02
3
+ experts:
4
+ - source_model: cognitivecomputations/dolphin-2.8-mistral-7b-v02
5
+ positive_prompts:
6
+ - "chat"
7
+ - "assistant"
8
+ - "tell me"
9
+ - "explain"
10
+ - "I want"
11
+ - source_model: OpenPipe/mistral-ft-optimized-1227
12
+ positive_prompts:
13
+ - "code"
14
+ - "python"
15
+ - "javascript"
16
+ - "programming"
17
+ - "algorithm"
model-00001-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8947d32631b22ab6ef5cc552cb3f918ce1c536906c84dd58f6ae4d051b59d08b
3
+ size 1933882680
model-00002-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5968f3922bf941fff173c698d41c460b4e45a12f81c894161fc7a2fc5aa19583
3
+ size 1996490960
model-00003-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d6e65e988ee274a859a6dba6bceac0d29de2876e395ba660d2facbb48054c2d
3
+ size 1996490968
model-00004-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8b4b4a0b6868d10ef6f1d372ad34da12f9448ce8e15bd3af4cb2b88b95b797ea
3
+ size 1996490968
model-00005-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e6d935aa8359b2aa4cd56cd75712138c9f367580467d3b6992523663bde5e9b
3
+ size 1996490952
model-00006-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:85dfe66217109dd2891abc649cfa7b60318efbd14fcb72582e54069f33204631
3
+ size 1996490960
model-00007-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3ab93145af7fb6ae2644f81538844dfa010ce8df307284db59d07a2e467fc9a
3
+ size 1996490968
model-00008-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6e26579ccf0a549360462ecb225ed973365c08c9feaa4031d16cfc4dbcc2fcf
3
+ size 1996490960
model-00009-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2521e67947dd85ae20204875ac4f5535e43ef97a2d604abbccbfe60d92995c94
3
+ size 1996490952
model-00010-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ba6548817fd563ecda962a90912347cf1021f36ecc84a32b3bca9afba30e0139
3
+ size 1996490968
model-00011-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6abf83269e393fd9fd581526fdb4b3f9d4007e6bdbdc72643f87353371f2009d
3
+ size 1996490968
model-00012-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f59c82c2accf55a05be49e5bd58b35ef2efd99e47491f9d5cf79f1a92c20d887
3
+ size 1979981568
model-00013-of-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:994a34c529e3c07a2e2a78db31c07115e51aadf2cad5b3f290f8d50273f3bb60
3
+ size 1879588416
model.safetensors.index.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"metadata": {"mergekit_version": "0.0.4"}, "weight_map": {"model.embed_tokens.weight": "model-00001-of-00013.safetensors", "model.norm.weight": "model-00001-of-00013.safetensors", "lm_head.weight": "model-00001-of-00013.safetensors", "model.layers.0.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.1.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.2.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.3.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.4.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.5.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.6.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.7.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.8.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.9.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.10.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.11.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.12.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.13.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.14.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.15.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.16.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.17.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.18.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.19.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.20.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.21.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.22.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.23.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.24.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.25.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.26.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.27.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.28.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.29.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.30.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.31.input_layernorm.weight": "model-00001-of-00013.safetensors", "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00013.safetensors", "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00013.safetensors", "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00013.safetensors", "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00013.safetensors", "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00013.safetensors", "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00013.safetensors", "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00013.safetensors", "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00013.safetensors", "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00013.safetensors", "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00013.safetensors", "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00013.safetensors", "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00013.safetensors", "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00013.safetensors", "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00013.safetensors", "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00013.safetensors", "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00013.safetensors", "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00013.safetensors", "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00013.safetensors", "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00013.safetensors", "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00013.safetensors", "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00013.safetensors", "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00013.safetensors", "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00013.safetensors", "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00013.safetensors", "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00013.safetensors", "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00013.safetensors", "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00013.safetensors", "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00013.safetensors", "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00013.safetensors", "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00013.safetensors", "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00013.safetensors", "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00013.safetensors", "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00013.safetensors", "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00013.safetensors", "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00013.safetensors", "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00013.safetensors", "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00013.safetensors", "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00013.safetensors", "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00013.safetensors", "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00013.safetensors", "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00013.safetensors", "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00013.safetensors", "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00013.safetensors", "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00013.safetensors", "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00013.safetensors", "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00013.safetensors", "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00013.safetensors", "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00013.safetensors", "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00013.safetensors", "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00013.safetensors", "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00013.safetensors", "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00013.safetensors", "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00013.safetensors", "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00013.safetensors", "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00013.safetensors", "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00013.safetensors", "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00013.safetensors", "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00013.safetensors", "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00013.safetensors", "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00013.safetensors", "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00013.safetensors", "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00013.safetensors", "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00013.safetensors", "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00013.safetensors", "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00013.safetensors", "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00013.safetensors", "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00013.safetensors", "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00013.safetensors", "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00013.safetensors", "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00013.safetensors", "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00013.safetensors", "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00013.safetensors", "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00013.safetensors", "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00013.safetensors", "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00013.safetensors", "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00013.safetensors", "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00013.safetensors", "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00013.safetensors", "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00013.safetensors", "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00013.safetensors", "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00013.safetensors", "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00013.safetensors", "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00013.safetensors", "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00013.safetensors", "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00013.safetensors", "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00013.safetensors", "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00013.safetensors", "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00013.safetensors", "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00013.safetensors", "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00013.safetensors", "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00013.safetensors", "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00013.safetensors", "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00013.safetensors", "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00013.safetensors", "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00013.safetensors", "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00013.safetensors", "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00013.safetensors", "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00013.safetensors", "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00013.safetensors", "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00013.safetensors", "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00013.safetensors", "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00013.safetensors", "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00013.safetensors", "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00013.safetensors", "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00013.safetensors", "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00013.safetensors", "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00013.safetensors", "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00013.safetensors", "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00013.safetensors", "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00013.safetensors", "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00013.safetensors", "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00013.safetensors", "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00013.safetensors", "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00013.safetensors", "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00013.safetensors", "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00013.safetensors", "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00013.safetensors", "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00013.safetensors", "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00013.safetensors", "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00013.safetensors", "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00013.safetensors", "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00013.safetensors", "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00013.safetensors", "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00013.safetensors", "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00013.safetensors", "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00013.safetensors", "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00013.safetensors", "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00013.safetensors", "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00008-of-00013.safetensors", "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00008-of-00013.safetensors", "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00008-of-00013.safetensors", "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00013.safetensors", "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00013.safetensors", "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00013.safetensors", "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00013.safetensors", "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00013.safetensors", "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00013.safetensors", "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00013.safetensors", "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00013.safetensors", "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00013.safetensors", "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00013.safetensors", "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00013.safetensors", "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00013.safetensors", "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00013.safetensors", "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00013.safetensors", "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00013.safetensors", "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00013.safetensors", "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00013.safetensors", "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00013.safetensors", "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00013.safetensors", "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00013.safetensors", "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00013.safetensors", "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00013.safetensors", "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00013.safetensors", "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00013.safetensors", "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00013.safetensors", "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00013.safetensors", "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00013.safetensors", "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00013.safetensors", "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00013.safetensors", "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00013.safetensors", "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00013.safetensors", "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00013.safetensors", "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00013.safetensors", "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00013.safetensors", "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00013.safetensors", "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00013.safetensors", "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00013.safetensors", "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00013.safetensors", "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00013.safetensors", "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00013.safetensors", "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00013.safetensors", "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00013.safetensors", "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00013.safetensors", "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00013.safetensors", "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00013.safetensors", "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00013.safetensors", "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00013.safetensors", "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00013.safetensors", "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00013.safetensors", "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00013.safetensors", "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00013.safetensors", "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00013.safetensors", "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00013.safetensors", "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00013.safetensors", "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00013.safetensors", "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00013.safetensors", "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00013.safetensors", "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00013.safetensors", "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00013.safetensors", "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00013.safetensors", "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00013.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.16.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.17.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.18.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.19.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.20.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.21.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.22.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.23.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.24.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.25.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.26.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.27.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.28.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.29.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.30.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.31.post_attention_layernorm.weight": "model-00012-of-00013.safetensors", "model.layers.0.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.1.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.2.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.3.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.4.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.5.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.6.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.7.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.8.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.9.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.10.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.11.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.12.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.13.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.14.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.15.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.16.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.17.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.18.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.19.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.20.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.21.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.22.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.23.self_attn.q_proj.weight": "model-00012-of-00013.safetensors", "model.layers.24.self_attn.q_proj.weight": "model-00013-of-00013.safetensors", "model.layers.25.self_attn.q_proj.weight": "model-00013-of-00013.safetensors", "model.layers.26.self_attn.q_proj.weight": "model-00013-of-00013.safetensors", "model.layers.27.self_attn.q_proj.weight": "model-00013-of-00013.safetensors", "model.layers.28.self_attn.q_proj.weight": "model-00013-of-00013.safetensors", "model.layers.29.self_attn.q_proj.weight": "model-00013-of-00013.safetensors", "model.layers.30.self_attn.q_proj.weight": "model-00013-of-00013.safetensors", "model.layers.31.self_attn.q_proj.weight": "model-00013-of-00013.safetensors", "model.layers.0.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.1.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.2.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.3.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.4.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.5.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.6.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.7.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.8.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.9.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.10.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.11.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.12.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.13.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.14.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.15.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.16.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.17.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.18.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.19.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.20.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.21.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.22.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.23.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.24.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.25.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.26.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.27.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.28.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.29.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.30.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.31.self_attn.k_proj.weight": "model-00013-of-00013.safetensors", "model.layers.0.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.1.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.2.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.3.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.4.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.5.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.6.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.7.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.8.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.9.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.10.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.11.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.12.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.13.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.14.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.15.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.16.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.17.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.18.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.19.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.20.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.21.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.22.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.23.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.24.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.25.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.26.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.27.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.28.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.29.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.30.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.31.self_attn.v_proj.weight": "model-00013-of-00013.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.10.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.11.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.12.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.13.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.14.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.15.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.16.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.17.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.18.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.19.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.20.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.21.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.22.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.23.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.24.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.25.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.26.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.27.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.28.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.29.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.30.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.31.self_attn.o_proj.weight": "model-00013-of-00013.safetensors", "model.layers.0.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.1.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.2.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.3.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.4.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.5.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.6.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.7.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.8.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.9.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.10.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.11.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.12.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.13.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.14.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.15.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.16.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.17.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.18.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.19.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.20.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.21.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.22.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.23.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.24.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.25.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.26.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.27.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.28.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.29.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.30.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors", "model.layers.31.block_sparse_moe.gate.weight": "model-00013-of-00013.safetensors"}}
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|im_end|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
+ size 493443
tokenizer_config.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ },
30
+ "32000": {
31
+ "content": "<|im_end|>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
+ "32001": {
39
+ "content": "<|im_start|>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": false,
43
+ "single_word": false,
44
+ "special": false
45
+ }
46
+ },
47
+ "bos_token": "<s>",
48
+ "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
49
+ "clean_up_tokenization_spaces": false,
50
+ "eos_token": "<|im_end|>",
51
+ "legacy": true,
52
+ "model_max_length": 1000000000000000019884624838656,
53
+ "pad_token": "<s>",
54
+ "sp_model_kwargs": {},
55
+ "spaces_between_special_tokens": false,
56
+ "tokenizer_class": "LlamaTokenizer",
57
+ "unk_token": "<unk>",
58
+ "use_default_system_prompt": false,
59
+ "use_fast": true
60
+ }