reciprocate
commited on
Commit
•
03b5bc9
1
Parent(s):
b5acbe8
update plot with new naming & reformat the code
Browse files
README.md
CHANGED
@@ -42,19 +42,23 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
42 |
|
43 |
tokenizer = AutoTokenizer.from_pretrained('stabilityai/stablelm-zephyr-3b')
|
44 |
model = AutoModelForCausalLM.from_pretrained(
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
)
|
49 |
|
50 |
prompt = [{'role': 'user', 'content': 'List 3 synonyms for the word "tiny"'}]
|
51 |
-
inputs = tokenizer.apply_chat_template(
|
|
|
|
|
|
|
|
|
52 |
|
53 |
tokens = model.generate(
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
)
|
59 |
|
60 |
print(tokenizer.decode(tokens[0], skip_special_tokens=False))
|
@@ -87,14 +91,15 @@ The dataset is comprised of a mixture of open datasets large-scale datasets avai
|
|
87 |
|
88 |
## Performance
|
89 |
|
90 |
-
### MT
|
91 |
|
92 |
-
<img src="https://cdn-uploads.huggingface.co/production/uploads/6310474ca119d49bc1eb0d80/
|
93 |
|
94 |
| Model | Size | Alignment | MT-Bench (score) | AlpacaEval (win rate %) |
|
95 |
|-------------|-----|----|---------------|--------------|
|
96 |
| **StableLM Zephyr 3B** 🪁 | 3B | DPO | 6.64 | 76.00 |
|
97 |
| Stable Zephyr (SFT only) | 3B | SFT | 6.04 | 71.15 |
|
|
|
98 |
| MPT-Chat | 7B |dSFT |5.42| -|
|
99 |
| Xwin-LMv0.1 | 7B| dPPO| 6.19| 87.83|
|
100 |
| Mistral-Instructv0.1 | 7B| - | 6.84 |-|
|
|
|
42 |
|
43 |
tokenizer = AutoTokenizer.from_pretrained('stabilityai/stablelm-zephyr-3b')
|
44 |
model = AutoModelForCausalLM.from_pretrained(
|
45 |
+
'stabilityai/stablelm-zephyr-3b',
|
46 |
+
trust_remote_code=True,
|
47 |
+
device_map="auto"
|
48 |
)
|
49 |
|
50 |
prompt = [{'role': 'user', 'content': 'List 3 synonyms for the word "tiny"'}]
|
51 |
+
inputs = tokenizer.apply_chat_template(
|
52 |
+
prompt,
|
53 |
+
add_generation_prompt=True,
|
54 |
+
return_tensors='pt'
|
55 |
+
)
|
56 |
|
57 |
tokens = model.generate(
|
58 |
+
inputs.to(model.device),
|
59 |
+
max_new_tokens=1024,
|
60 |
+
temperature=0.8,
|
61 |
+
do_sample=True
|
62 |
)
|
63 |
|
64 |
print(tokenizer.decode(tokens[0], skip_special_tokens=False))
|
|
|
91 |
|
92 |
## Performance
|
93 |
|
94 |
+
### MT-Bench and Alpaca Bench
|
95 |
|
96 |
+
<img src="https://cdn-uploads.huggingface.co/production/uploads/6310474ca119d49bc1eb0d80/jwpbBHzdCkHm0rMvPUVxC.png" alt="mt_bench_plot" width="600"/>
|
97 |
|
98 |
| Model | Size | Alignment | MT-Bench (score) | AlpacaEval (win rate %) |
|
99 |
|-------------|-----|----|---------------|--------------|
|
100 |
| **StableLM Zephyr 3B** 🪁 | 3B | DPO | 6.64 | 76.00 |
|
101 |
| Stable Zephyr (SFT only) | 3B | SFT | 6.04 | 71.15 |
|
102 |
+
| Capybara v1.9 | 3B | dSFT | 5.94 | - |
|
103 |
| MPT-Chat | 7B |dSFT |5.42| -|
|
104 |
| Xwin-LMv0.1 | 7B| dPPO| 6.19| 87.83|
|
105 |
| Mistral-Instructv0.1 | 7B| - | 6.84 |-|
|