Update README.md
Browse files
README.md
CHANGED
@@ -21,10 +21,10 @@ model-index:
|
|
21 |
metrics:
|
22 |
- name: Accuracy (0-shot)
|
23 |
type: Accuracy (0-shot)
|
24 |
-
value:
|
25 |
source:
|
26 |
name: Paper
|
27 |
-
url: https://
|
28 |
- task:
|
29 |
type: multiple-choice
|
30 |
dataset:
|
@@ -33,10 +33,10 @@ model-index:
|
|
33 |
metrics:
|
34 |
- name: Accuracy (5-shot)
|
35 |
type: Accuracy (5-shot)
|
36 |
-
value:
|
37 |
source:
|
38 |
name: Paper
|
39 |
-
url: https://
|
40 |
- task:
|
41 |
type: mix
|
42 |
dataset:
|
@@ -45,10 +45,10 @@ model-index:
|
|
45 |
metrics:
|
46 |
- name: Average scores (5-shot)
|
47 |
type: Average scores (5-shot)
|
48 |
-
value:
|
49 |
source:
|
50 |
name: Paper
|
51 |
-
url: https://
|
52 |
- task:
|
53 |
type: multiple_choice
|
54 |
dataset:
|
@@ -57,10 +57,10 @@ model-index:
|
|
57 |
metrics:
|
58 |
- name: Accuracy (5-shot)
|
59 |
type: Accuracy (5-shot)
|
60 |
-
value:
|
61 |
source:
|
62 |
name: Paper
|
63 |
-
url: https://
|
64 |
- task:
|
65 |
type: multiple_choice
|
66 |
dataset:
|
@@ -69,10 +69,10 @@ model-index:
|
|
69 |
metrics:
|
70 |
- name: Accuracy (5-shot)
|
71 |
type: Accuracy (5-shot)
|
72 |
-
value:
|
73 |
source:
|
74 |
name: Paper
|
75 |
-
url: https://
|
76 |
- task:
|
77 |
type: multiple_choice
|
78 |
dataset:
|
@@ -81,10 +81,10 @@ model-index:
|
|
81 |
metrics:
|
82 |
- name: Accuracy (5-shot)
|
83 |
type: Accuracy (5-shot)
|
84 |
-
value: 56.
|
85 |
source:
|
86 |
name: Paper
|
87 |
-
url: https://
|
88 |
- task:
|
89 |
type: multiple_choice
|
90 |
dataset:
|
@@ -93,10 +93,10 @@ model-index:
|
|
93 |
metrics:
|
94 |
- name: Accuracy (5-shot)
|
95 |
type: Accuracy (5-shot)
|
96 |
-
value: 43.
|
97 |
source:
|
98 |
name: Paper
|
99 |
-
url: https://
|
100 |
---
|
101 |
|
102 |
# **Model Card for Latxa 13b**
|
@@ -251,21 +251,18 @@ In order to reproduce our results please follow the instructions in Latxa's [Git
|
|
251 |
|
|
252 |
| XGLM | 7B | 57.71 | 23.88 | 41.47 | 22.96 | 24.43 | 26.53 | 24.59 | 32.51 |
|
253 |
| BLOOM | 7B | 57.18 | 27.00 | 40.17 | 25.34 | 28.41 | 27.17 | 25.07 | 33.86 |
|
254 |
-
| Mistral | 7B | 51.09 | **38.89**| 39.22 | 25.01 | 29.26 | 34.58 | 32.15 | 35.94 |
|
255 |
| Llama 2 | 7B | 50.43 | 26.22 | 38.20 | 24.09 | 27.27 | 29.50 | 28.84 | 32.51 |
|
256 |
-
| **Latxa v1**
|
257 |
-
| **Latxa v1.1** | 7B | **65.72**| 36.89 | **51.78**| **32.44**| **30.40**| **44.37** | **34.20**| **42.26** |
|
258 |
|
|
259 |
| mGPT | 13B | 55.39 | 25.00 | 37.56 | 25.00 | 24.15 | 27.17 | 25.73 | 32.14 |
|
260 |
| Llama 2 | 13B | 50.63 | 32.00 | 38.98 | 25.90 | 28.98 | 33.53 | 29.66 | 34.36 |
|
261 |
-
| **Latxa v1**
|
262 |
-
| **Latxa v1.1** | 13B | **67.24**| 51.56 | 54.04 | **45.02**| 29.83 | **56.44** | **43.18**| 49.62 |
|
263 |
|
|
264 |
| Mixtral | 8x7B | 52.55 | 50.44 | 45.00 | 26.43 | 37.50 | 42.51 | 39.87 | 41.97 |
|
265 |
| Yi | 34B | 52.22 | 54.56 | 43.90 | 27.30 | 34.66 | 42.57 | 39.68 | 42.05 |
|
266 |
| Llama 2 | 70B | 51.62 | 33.56 | 42.55 | 24.16 | 27.84 | 38.43 | 33.08 | 35.47 |
|
267 |
-
| **Latxa v1**
|
268 |
-
| **Latxa v1.1** | 70B | **69.76**| 64.89| **61.66**| **60.61**| **53.69**| **61.52** | **54.48**| **60.94** |
|
269 |
|
270 |
|
271 |
# **Environmental Impact**
|
|
|
21 |
metrics:
|
22 |
- name: Accuracy (0-shot)
|
23 |
type: Accuracy (0-shot)
|
24 |
+
value: 65.51
|
25 |
source:
|
26 |
name: Paper
|
27 |
+
url: https://arxiv.org/abs/2403.20266
|
28 |
- task:
|
29 |
type: multiple-choice
|
30 |
dataset:
|
|
|
33 |
metrics:
|
34 |
- name: Accuracy (5-shot)
|
35 |
type: Accuracy (5-shot)
|
36 |
+
value: 53.89
|
37 |
source:
|
38 |
name: Paper
|
39 |
+
url: https://arxiv.org/abs/2403.20266
|
40 |
- task:
|
41 |
type: mix
|
42 |
dataset:
|
|
|
45 |
metrics:
|
46 |
- name: Average scores (5-shot)
|
47 |
type: Average scores (5-shot)
|
48 |
+
value: 53.56
|
49 |
source:
|
50 |
name: Paper
|
51 |
+
url: https://arxiv.org/abs/2403.20266
|
52 |
- task:
|
53 |
type: multiple_choice
|
54 |
dataset:
|
|
|
57 |
metrics:
|
58 |
- name: Accuracy (5-shot)
|
59 |
type: Accuracy (5-shot)
|
60 |
+
value: 44.11
|
61 |
source:
|
62 |
name: Paper
|
63 |
+
url: https://arxiv.org/abs/2403.20266
|
64 |
- task:
|
65 |
type: multiple_choice
|
66 |
dataset:
|
|
|
69 |
metrics:
|
70 |
- name: Accuracy (5-shot)
|
71 |
type: Accuracy (5-shot)
|
72 |
+
value: 32.67
|
73 |
source:
|
74 |
name: Paper
|
75 |
+
url: https://arxiv.org/abs/2403.20266
|
76 |
- task:
|
77 |
type: multiple_choice
|
78 |
dataset:
|
|
|
81 |
metrics:
|
82 |
- name: Accuracy (5-shot)
|
83 |
type: Accuracy (5-shot)
|
84 |
+
value: 56.38
|
85 |
source:
|
86 |
name: Paper
|
87 |
+
url: https://arxiv.org/abs/2403.20266
|
88 |
- task:
|
89 |
type: multiple_choice
|
90 |
dataset:
|
|
|
93 |
metrics:
|
94 |
- name: Accuracy (5-shot)
|
95 |
type: Accuracy (5-shot)
|
96 |
+
value: 43.66
|
97 |
source:
|
98 |
name: Paper
|
99 |
+
url: https://arxiv.org/abs/2403.20266
|
100 |
---
|
101 |
|
102 |
# **Model Card for Latxa 13b**
|
|
|
251 |
|
|
252 |
| XGLM | 7B | 57.71 | 23.88 | 41.47 | 22.96 | 24.43 | 26.53 | 24.59 | 32.51 |
|
253 |
| BLOOM | 7B | 57.18 | 27.00 | 40.17 | 25.34 | 28.41 | 27.17 | 25.07 | 33.86 |
|
254 |
+
| Mistral | 7B | 51.09 | **38.89**| 39.22 | 25.01 | **29.26** | 34.58 | 32.15 | 35.94 |
|
255 |
| Llama 2 | 7B | 50.43 | 26.22 | 38.20 | 24.09 | 27.27 | 29.50 | 28.84 | 32.51 |
|
256 |
+
| **Latxa v1.1** | 7B | **65.45**| 37.33 | **52.56**| **30.26**| 25.00| **42.16** | **33.82**| **40.94** |
|
|
|
257 |
|
|
258 |
| mGPT | 13B | 55.39 | 25.00 | 37.56 | 25.00 | 24.15 | 27.17 | 25.73 | 32.14 |
|
259 |
| Llama 2 | 13B | 50.63 | 32.00 | 38.98 | 25.90 | 28.98 | 33.53 | 29.66 | 34.36 |
|
260 |
+
| **Latxa v1.1** | 13B | **66.51**| **53.89** | **53.36** | **44.11**| **32.67** | **56.38** | **43.66**| **50.08** |
|
|
|
261 |
|
|
262 |
| Mixtral | 8x7B | 52.55 | 50.44 | 45.00 | 26.43 | 37.50 | 42.51 | 39.87 | 41.97 |
|
263 |
| Yi | 34B | 52.22 | 54.56 | 43.90 | 27.30 | 34.66 | 42.57 | 39.68 | 42.05 |
|
264 |
| Llama 2 | 70B | 51.62 | 33.56 | 42.55 | 24.16 | 27.84 | 38.43 | 33.08 | 35.47 |
|
265 |
+
| **Latxa v1.1** | 70B | **70.55**| **71.67** | **59.74**| **60.65**| **50.57**| **62.45** | **51.90**| **61.08** |
|
|
|
266 |
|
267 |
|
268 |
# **Environmental Impact**
|