|
--- |
|
tags: |
|
- mteb |
|
- transformers |
|
- sentence-transformers |
|
model-index: |
|
- name: Linq-Embed-Mistral |
|
results: |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_counterfactual |
|
name: MTEB AmazonCounterfactualClassification (en) |
|
config: en |
|
split: test |
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
metrics: |
|
- type: accuracy |
|
value: 84.43283582089552 |
|
- type: ap |
|
value: 50.39222584035829 |
|
- type: f1 |
|
value: 78.47906270064071 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_polarity |
|
name: MTEB AmazonPolarityClassification |
|
config: default |
|
split: test |
|
revision: e2d317d38cd51312af73b3d32a06d1a08b442046 |
|
metrics: |
|
- type: accuracy |
|
value: 95.70445 |
|
- type: ap |
|
value: 94.28273900595173 |
|
- type: f1 |
|
value: 95.70048412173735 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (en) |
|
config: en |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 57.644000000000005 |
|
- type: f1 |
|
value: 56.993648296704876 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/arguana |
|
name: MTEB ArguAna |
|
config: default |
|
split: test |
|
revision: c22ab2a51041ffd869aaddef7af8d8215647e41a |
|
metrics: |
|
- type: map_at_1 |
|
value: 45.804 |
|
- type: map_at_10 |
|
value: 61.742 |
|
- type: map_at_100 |
|
value: 62.07899999999999 |
|
- type: map_at_1000 |
|
value: 62.08 |
|
- type: map_at_3 |
|
value: 57.717 |
|
- type: map_at_5 |
|
value: 60.27 |
|
- type: mrr_at_1 |
|
value: 47.226 |
|
- type: mrr_at_10 |
|
value: 62.256 |
|
- type: mrr_at_100 |
|
value: 62.601 |
|
- type: mrr_at_1000 |
|
value: 62.601 |
|
- type: mrr_at_3 |
|
value: 58.203 |
|
- type: mrr_at_5 |
|
value: 60.767 |
|
- type: ndcg_at_1 |
|
value: 45.804 |
|
- type: ndcg_at_10 |
|
value: 69.649 |
|
- type: ndcg_at_100 |
|
value: 70.902 |
|
- type: ndcg_at_1000 |
|
value: 70.91199999999999 |
|
- type: ndcg_at_3 |
|
value: 61.497 |
|
- type: ndcg_at_5 |
|
value: 66.097 |
|
- type: precision_at_1 |
|
value: 45.804 |
|
- type: precision_at_10 |
|
value: 9.452 |
|
- type: precision_at_100 |
|
value: 0.996 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_3 |
|
value: 24.135 |
|
- type: precision_at_5 |
|
value: 16.714000000000002 |
|
- type: recall_at_1 |
|
value: 45.804 |
|
- type: recall_at_10 |
|
value: 94.523 |
|
- type: recall_at_100 |
|
value: 99.57300000000001 |
|
- type: recall_at_1000 |
|
value: 99.644 |
|
- type: recall_at_3 |
|
value: 72.404 |
|
- type: recall_at_5 |
|
value: 83.57 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/arxiv-clustering-p2p |
|
name: MTEB ArxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d |
|
metrics: |
|
- type: v_measure |
|
value: 51.47612678878609 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/arxiv-clustering-s2s |
|
name: MTEB ArxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 |
|
metrics: |
|
- type: v_measure |
|
value: 47.2977392340418 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/askubuntudupquestions-reranking |
|
name: MTEB AskUbuntuDupQuestions |
|
config: default |
|
split: test |
|
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 |
|
metrics: |
|
- type: map |
|
value: 66.82016765243456 |
|
- type: mrr |
|
value: 79.55227982236292 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/biosses-sts |
|
name: MTEB BIOSSES |
|
config: default |
|
split: test |
|
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 89.15068664186332 |
|
- type: cos_sim_spearman |
|
value: 86.4013663041054 |
|
- type: euclidean_pearson |
|
value: 87.36391302921588 |
|
- type: euclidean_spearman |
|
value: 86.4013663041054 |
|
- type: manhattan_pearson |
|
value: 87.46116676558589 |
|
- type: manhattan_spearman |
|
value: 86.78149544753352 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/banking77 |
|
name: MTEB Banking77Classification |
|
config: default |
|
split: test |
|
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 |
|
metrics: |
|
- type: accuracy |
|
value: 87.88311688311688 |
|
- type: f1 |
|
value: 87.82368154811464 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/biorxiv-clustering-p2p |
|
name: MTEB BiorxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 |
|
metrics: |
|
- type: v_measure |
|
value: 42.72860396750569 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/biorxiv-clustering-s2s |
|
name: MTEB BiorxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 |
|
metrics: |
|
- type: v_measure |
|
value: 39.58412067938718 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/cqadupstack |
|
name: MTEB CQADupstackRetrieval |
|
config: default |
|
split: test |
|
revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 |
|
metrics: |
|
- type: map_at_1 |
|
value: 30.082666666666665 |
|
- type: map_at_10 |
|
value: 41.13875 |
|
- type: map_at_100 |
|
value: 42.45525 |
|
- type: map_at_1000 |
|
value: 42.561249999999994 |
|
- type: map_at_3 |
|
value: 37.822750000000006 |
|
- type: map_at_5 |
|
value: 39.62658333333333 |
|
- type: mrr_at_1 |
|
value: 35.584 |
|
- type: mrr_at_10 |
|
value: 45.4675 |
|
- type: mrr_at_100 |
|
value: 46.31016666666667 |
|
- type: mrr_at_1000 |
|
value: 46.35191666666666 |
|
- type: mrr_at_3 |
|
value: 42.86674999999999 |
|
- type: mrr_at_5 |
|
value: 44.31341666666666 |
|
- type: ndcg_at_1 |
|
value: 35.584 |
|
- type: ndcg_at_10 |
|
value: 47.26516666666667 |
|
- type: ndcg_at_100 |
|
value: 52.49108333333332 |
|
- type: ndcg_at_1000 |
|
value: 54.24575 |
|
- type: ndcg_at_3 |
|
value: 41.83433333333334 |
|
- type: ndcg_at_5 |
|
value: 44.29899999999999 |
|
- type: precision_at_1 |
|
value: 35.584 |
|
- type: precision_at_10 |
|
value: 8.390333333333334 |
|
- type: precision_at_100 |
|
value: 1.2941666666666667 |
|
- type: precision_at_1000 |
|
value: 0.16308333333333336 |
|
- type: precision_at_3 |
|
value: 19.414583333333333 |
|
- type: precision_at_5 |
|
value: 13.751 |
|
- type: recall_at_1 |
|
value: 30.082666666666665 |
|
- type: recall_at_10 |
|
value: 60.88875 |
|
- type: recall_at_100 |
|
value: 83.35141666666667 |
|
- type: recall_at_1000 |
|
value: 95.0805 |
|
- type: recall_at_3 |
|
value: 45.683749999999996 |
|
- type: recall_at_5 |
|
value: 52.08208333333333 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/climate-fever |
|
name: MTEB ClimateFEVER |
|
config: default |
|
split: test |
|
revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 |
|
metrics: |
|
- type: map_at_1 |
|
value: 16.747 |
|
- type: map_at_10 |
|
value: 29.168 |
|
- type: map_at_100 |
|
value: 31.304 |
|
- type: map_at_1000 |
|
value: 31.496000000000002 |
|
- type: map_at_3 |
|
value: 24.57 |
|
- type: map_at_5 |
|
value: 26.886 |
|
- type: mrr_at_1 |
|
value: 37.524 |
|
- type: mrr_at_10 |
|
value: 50.588 |
|
- type: mrr_at_100 |
|
value: 51.28 |
|
- type: mrr_at_1000 |
|
value: 51.29899999999999 |
|
- type: mrr_at_3 |
|
value: 47.438 |
|
- type: mrr_at_5 |
|
value: 49.434 |
|
- type: ndcg_at_1 |
|
value: 37.524 |
|
- type: ndcg_at_10 |
|
value: 39.11 |
|
- type: ndcg_at_100 |
|
value: 46.373999999999995 |
|
- type: ndcg_at_1000 |
|
value: 49.370999999999995 |
|
- type: ndcg_at_3 |
|
value: 32.964 |
|
- type: ndcg_at_5 |
|
value: 35.028 |
|
- type: precision_at_1 |
|
value: 37.524 |
|
- type: precision_at_10 |
|
value: 12.137 |
|
- type: precision_at_100 |
|
value: 1.9929999999999999 |
|
- type: precision_at_1000 |
|
value: 0.256 |
|
- type: precision_at_3 |
|
value: 24.886 |
|
- type: precision_at_5 |
|
value: 18.762 |
|
- type: recall_at_1 |
|
value: 16.747 |
|
- type: recall_at_10 |
|
value: 45.486 |
|
- type: recall_at_100 |
|
value: 69.705 |
|
- type: recall_at_1000 |
|
value: 86.119 |
|
- type: recall_at_3 |
|
value: 30.070999999999998 |
|
- type: recall_at_5 |
|
value: 36.565 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/dbpedia |
|
name: MTEB DBPedia |
|
config: default |
|
split: test |
|
revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 |
|
metrics: |
|
- type: map_at_1 |
|
value: 10.495000000000001 |
|
- type: map_at_10 |
|
value: 24.005000000000003 |
|
- type: map_at_100 |
|
value: 34.37 |
|
- type: map_at_1000 |
|
value: 36.268 |
|
- type: map_at_3 |
|
value: 16.694 |
|
- type: map_at_5 |
|
value: 19.845 |
|
- type: mrr_at_1 |
|
value: 75.5 |
|
- type: mrr_at_10 |
|
value: 82.458 |
|
- type: mrr_at_100 |
|
value: 82.638 |
|
- type: mrr_at_1000 |
|
value: 82.64 |
|
- type: mrr_at_3 |
|
value: 81.25 |
|
- type: mrr_at_5 |
|
value: 82.125 |
|
- type: ndcg_at_1 |
|
value: 64.625 |
|
- type: ndcg_at_10 |
|
value: 51.322 |
|
- type: ndcg_at_100 |
|
value: 55.413999999999994 |
|
- type: ndcg_at_1000 |
|
value: 62.169 |
|
- type: ndcg_at_3 |
|
value: 56.818999999999996 |
|
- type: ndcg_at_5 |
|
value: 54.32900000000001 |
|
- type: precision_at_1 |
|
value: 75.5 |
|
- type: precision_at_10 |
|
value: 40.849999999999994 |
|
- type: precision_at_100 |
|
value: 12.882 |
|
- type: precision_at_1000 |
|
value: 2.394 |
|
- type: precision_at_3 |
|
value: 59.667 |
|
- type: precision_at_5 |
|
value: 52.2 |
|
- type: recall_at_1 |
|
value: 10.495000000000001 |
|
- type: recall_at_10 |
|
value: 29.226000000000003 |
|
- type: recall_at_100 |
|
value: 59.614 |
|
- type: recall_at_1000 |
|
value: 81.862 |
|
- type: recall_at_3 |
|
value: 17.97 |
|
- type: recall_at_5 |
|
value: 22.438 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/emotion |
|
name: MTEB EmotionClassification |
|
config: default |
|
split: test |
|
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 |
|
metrics: |
|
- type: accuracy |
|
value: 51.82 |
|
- type: f1 |
|
value: 47.794956731921054 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/fever |
|
name: MTEB FEVER |
|
config: default |
|
split: test |
|
revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 |
|
metrics: |
|
- type: map_at_1 |
|
value: 82.52199999999999 |
|
- type: map_at_10 |
|
value: 89.794 |
|
- type: map_at_100 |
|
value: 89.962 |
|
- type: map_at_1000 |
|
value: 89.972 |
|
- type: map_at_3 |
|
value: 88.95100000000001 |
|
- type: map_at_5 |
|
value: 89.524 |
|
- type: mrr_at_1 |
|
value: 88.809 |
|
- type: mrr_at_10 |
|
value: 93.554 |
|
- type: mrr_at_100 |
|
value: 93.577 |
|
- type: mrr_at_1000 |
|
value: 93.577 |
|
- type: mrr_at_3 |
|
value: 93.324 |
|
- type: mrr_at_5 |
|
value: 93.516 |
|
- type: ndcg_at_1 |
|
value: 88.809 |
|
- type: ndcg_at_10 |
|
value: 92.419 |
|
- type: ndcg_at_100 |
|
value: 92.95 |
|
- type: ndcg_at_1000 |
|
value: 93.10000000000001 |
|
- type: ndcg_at_3 |
|
value: 91.45299999999999 |
|
- type: ndcg_at_5 |
|
value: 92.05 |
|
- type: precision_at_1 |
|
value: 88.809 |
|
- type: precision_at_10 |
|
value: 10.911999999999999 |
|
- type: precision_at_100 |
|
value: 1.143 |
|
- type: precision_at_1000 |
|
value: 0.117 |
|
- type: precision_at_3 |
|
value: 34.623 |
|
- type: precision_at_5 |
|
value: 21.343999999999998 |
|
- type: recall_at_1 |
|
value: 82.52199999999999 |
|
- type: recall_at_10 |
|
value: 96.59400000000001 |
|
- type: recall_at_100 |
|
value: 98.55699999999999 |
|
- type: recall_at_1000 |
|
value: 99.413 |
|
- type: recall_at_3 |
|
value: 94.02199999999999 |
|
- type: recall_at_5 |
|
value: 95.582 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/fiqa |
|
name: MTEB FiQA2018 |
|
config: default |
|
split: test |
|
revision: 27a168819829fe9bcd655c2df245fb19452e8e06 |
|
metrics: |
|
- type: map_at_1 |
|
value: 32.842 |
|
- type: map_at_10 |
|
value: 53.147 |
|
- type: map_at_100 |
|
value: 55.265 |
|
- type: map_at_1000 |
|
value: 55.37 |
|
- type: map_at_3 |
|
value: 46.495 |
|
- type: map_at_5 |
|
value: 50.214999999999996 |
|
- type: mrr_at_1 |
|
value: 61.574 |
|
- type: mrr_at_10 |
|
value: 68.426 |
|
- type: mrr_at_100 |
|
value: 68.935 |
|
- type: mrr_at_1000 |
|
value: 68.95400000000001 |
|
- type: mrr_at_3 |
|
value: 66.307 |
|
- type: mrr_at_5 |
|
value: 67.611 |
|
- type: ndcg_at_1 |
|
value: 61.574 |
|
- type: ndcg_at_10 |
|
value: 61.205 |
|
- type: ndcg_at_100 |
|
value: 67.25999999999999 |
|
- type: ndcg_at_1000 |
|
value: 68.657 |
|
- type: ndcg_at_3 |
|
value: 56.717 |
|
- type: ndcg_at_5 |
|
value: 58.196999999999996 |
|
- type: precision_at_1 |
|
value: 61.574 |
|
- type: precision_at_10 |
|
value: 16.852 |
|
- type: precision_at_100 |
|
value: 2.33 |
|
- type: precision_at_1000 |
|
value: 0.256 |
|
- type: precision_at_3 |
|
value: 37.5 |
|
- type: precision_at_5 |
|
value: 27.468999999999998 |
|
- type: recall_at_1 |
|
value: 32.842 |
|
- type: recall_at_10 |
|
value: 68.157 |
|
- type: recall_at_100 |
|
value: 89.5 |
|
- type: recall_at_1000 |
|
value: 97.68599999999999 |
|
- type: recall_at_3 |
|
value: 50.783 |
|
- type: recall_at_5 |
|
value: 58.672000000000004 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/hotpotqa |
|
name: MTEB HotpotQA |
|
config: default |
|
split: test |
|
revision: ab518f4d6fcca38d87c25209f94beba119d02014 |
|
metrics: |
|
- type: map_at_1 |
|
value: 39.068000000000005 |
|
- type: map_at_10 |
|
value: 69.253 |
|
- type: map_at_100 |
|
value: 70.036 |
|
- type: map_at_1000 |
|
value: 70.081 |
|
- type: map_at_3 |
|
value: 65.621 |
|
- type: map_at_5 |
|
value: 67.976 |
|
- type: mrr_at_1 |
|
value: 78.13600000000001 |
|
- type: mrr_at_10 |
|
value: 84.328 |
|
- type: mrr_at_100 |
|
value: 84.515 |
|
- type: mrr_at_1000 |
|
value: 84.52300000000001 |
|
- type: mrr_at_3 |
|
value: 83.52199999999999 |
|
- type: mrr_at_5 |
|
value: 84.019 |
|
- type: ndcg_at_1 |
|
value: 78.13600000000001 |
|
- type: ndcg_at_10 |
|
value: 76.236 |
|
- type: ndcg_at_100 |
|
value: 78.891 |
|
- type: ndcg_at_1000 |
|
value: 79.73400000000001 |
|
- type: ndcg_at_3 |
|
value: 71.258 |
|
- type: ndcg_at_5 |
|
value: 74.129 |
|
- type: precision_at_1 |
|
value: 78.13600000000001 |
|
- type: precision_at_10 |
|
value: 16.347 |
|
- type: precision_at_100 |
|
value: 1.839 |
|
- type: precision_at_1000 |
|
value: 0.19499999999999998 |
|
- type: precision_at_3 |
|
value: 47.189 |
|
- type: precision_at_5 |
|
value: 30.581999999999997 |
|
- type: recall_at_1 |
|
value: 39.068000000000005 |
|
- type: recall_at_10 |
|
value: 81.735 |
|
- type: recall_at_100 |
|
value: 91.945 |
|
- type: recall_at_1000 |
|
value: 97.44800000000001 |
|
- type: recall_at_3 |
|
value: 70.783 |
|
- type: recall_at_5 |
|
value: 76.455 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/imdb |
|
name: MTEB ImdbClassification |
|
config: default |
|
split: test |
|
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 |
|
metrics: |
|
- type: accuracy |
|
value: 94.7764 |
|
- type: ap |
|
value: 92.67841294818406 |
|
- type: f1 |
|
value: 94.77375157383646 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/msmarco |
|
name: MTEB MSMARCO |
|
config: default |
|
split: dev |
|
revision: c5a29a104738b98a9e76336939199e264163d4a0 |
|
metrics: |
|
- type: map_at_1 |
|
value: 24.624 |
|
- type: map_at_10 |
|
value: 37.861 |
|
- type: map_at_100 |
|
value: 39.011 |
|
- type: map_at_1000 |
|
value: 39.052 |
|
- type: map_at_3 |
|
value: 33.76 |
|
- type: map_at_5 |
|
value: 36.153 |
|
- type: mrr_at_1 |
|
value: 25.358000000000004 |
|
- type: mrr_at_10 |
|
value: 38.5 |
|
- type: mrr_at_100 |
|
value: 39.572 |
|
- type: mrr_at_1000 |
|
value: 39.607 |
|
- type: mrr_at_3 |
|
value: 34.491 |
|
- type: mrr_at_5 |
|
value: 36.83 |
|
- type: ndcg_at_1 |
|
value: 25.358000000000004 |
|
- type: ndcg_at_10 |
|
value: 45.214999999999996 |
|
- type: ndcg_at_100 |
|
value: 50.56 |
|
- type: ndcg_at_1000 |
|
value: 51.507999999999996 |
|
- type: ndcg_at_3 |
|
value: 36.925999999999995 |
|
- type: ndcg_at_5 |
|
value: 41.182 |
|
- type: precision_at_1 |
|
value: 25.358000000000004 |
|
- type: precision_at_10 |
|
value: 7.090000000000001 |
|
- type: precision_at_100 |
|
value: 0.9740000000000001 |
|
- type: precision_at_1000 |
|
value: 0.106 |
|
- type: precision_at_3 |
|
value: 15.697 |
|
- type: precision_at_5 |
|
value: 11.599 |
|
- type: recall_at_1 |
|
value: 24.624 |
|
- type: recall_at_10 |
|
value: 67.78699999999999 |
|
- type: recall_at_100 |
|
value: 92.11200000000001 |
|
- type: recall_at_1000 |
|
value: 99.208 |
|
- type: recall_at_3 |
|
value: 45.362 |
|
- type: recall_at_5 |
|
value: 55.58 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_domain |
|
name: MTEB MTOPDomainClassification (en) |
|
config: en |
|
split: test |
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
metrics: |
|
- type: accuracy |
|
value: 96.83310533515733 |
|
- type: f1 |
|
value: 96.57069781347995 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_intent |
|
name: MTEB MTOPIntentClassification (en) |
|
config: en |
|
split: test |
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
metrics: |
|
- type: accuracy |
|
value: 89.5690834473324 |
|
- type: f1 |
|
value: 73.7275204564728 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (en) |
|
config: en |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 82.67316745124411 |
|
- type: f1 |
|
value: 79.70626515721662 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (en) |
|
config: en |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 85.01344989912575 |
|
- type: f1 |
|
value: 84.45181022816965 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/medrxiv-clustering-p2p |
|
name: MTEB MedrxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 |
|
metrics: |
|
- type: v_measure |
|
value: 37.843426126777295 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/medrxiv-clustering-s2s |
|
name: MTEB MedrxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 |
|
metrics: |
|
- type: v_measure |
|
value: 36.651728547241476 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/mind_small |
|
name: MTEB MindSmallReranking |
|
config: default |
|
split: test |
|
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 |
|
metrics: |
|
- type: map |
|
value: 32.05750522793288 |
|
- type: mrr |
|
value: 33.28067556869468 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/nfcorpus |
|
name: MTEB NFCorpus |
|
config: default |
|
split: test |
|
revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 |
|
metrics: |
|
- type: map_at_1 |
|
value: 6.744 |
|
- type: map_at_10 |
|
value: 16.235 |
|
- type: map_at_100 |
|
value: 20.767 |
|
- type: map_at_1000 |
|
value: 22.469 |
|
- type: map_at_3 |
|
value: 11.708 |
|
- type: map_at_5 |
|
value: 13.924 |
|
- type: mrr_at_1 |
|
value: 55.728 |
|
- type: mrr_at_10 |
|
value: 63.869 |
|
- type: mrr_at_100 |
|
value: 64.322 |
|
- type: mrr_at_1000 |
|
value: 64.342 |
|
- type: mrr_at_3 |
|
value: 62.022999999999996 |
|
- type: mrr_at_5 |
|
value: 63.105999999999995 |
|
- type: ndcg_at_1 |
|
value: 53.096 |
|
- type: ndcg_at_10 |
|
value: 41.618 |
|
- type: ndcg_at_100 |
|
value: 38.562999999999995 |
|
- type: ndcg_at_1000 |
|
value: 47.006 |
|
- type: ndcg_at_3 |
|
value: 47.657 |
|
- type: ndcg_at_5 |
|
value: 45.562999999999995 |
|
- type: precision_at_1 |
|
value: 55.108000000000004 |
|
- type: precision_at_10 |
|
value: 30.464000000000002 |
|
- type: precision_at_100 |
|
value: 9.737 |
|
- type: precision_at_1000 |
|
value: 2.2720000000000002 |
|
- type: precision_at_3 |
|
value: 44.376 |
|
- type: precision_at_5 |
|
value: 39.505 |
|
- type: recall_at_1 |
|
value: 6.744 |
|
- type: recall_at_10 |
|
value: 21.11 |
|
- type: recall_at_100 |
|
value: 39.69 |
|
- type: recall_at_1000 |
|
value: 70.44 |
|
- type: recall_at_3 |
|
value: 13.120000000000001 |
|
- type: recall_at_5 |
|
value: 16.669 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/nq |
|
name: MTEB NQ |
|
config: default |
|
split: test |
|
revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 |
|
metrics: |
|
- type: map_at_1 |
|
value: 46.263 |
|
- type: map_at_10 |
|
value: 63.525 |
|
- type: map_at_100 |
|
value: 64.142 |
|
- type: map_at_1000 |
|
value: 64.14800000000001 |
|
- type: map_at_3 |
|
value: 59.653 |
|
- type: map_at_5 |
|
value: 62.244 |
|
- type: mrr_at_1 |
|
value: 51.796 |
|
- type: mrr_at_10 |
|
value: 65.764 |
|
- type: mrr_at_100 |
|
value: 66.155 |
|
- type: mrr_at_1000 |
|
value: 66.158 |
|
- type: mrr_at_3 |
|
value: 63.05500000000001 |
|
- type: mrr_at_5 |
|
value: 64.924 |
|
- type: ndcg_at_1 |
|
value: 51.766999999999996 |
|
- type: ndcg_at_10 |
|
value: 70.626 |
|
- type: ndcg_at_100 |
|
value: 72.905 |
|
- type: ndcg_at_1000 |
|
value: 73.021 |
|
- type: ndcg_at_3 |
|
value: 63.937999999999995 |
|
- type: ndcg_at_5 |
|
value: 68.00699999999999 |
|
- type: precision_at_1 |
|
value: 51.766999999999996 |
|
- type: precision_at_10 |
|
value: 10.768 |
|
- type: precision_at_100 |
|
value: 1.203 |
|
- type: precision_at_1000 |
|
value: 0.121 |
|
- type: precision_at_3 |
|
value: 28.409000000000002 |
|
- type: precision_at_5 |
|
value: 19.502 |
|
- type: recall_at_1 |
|
value: 46.263 |
|
- type: recall_at_10 |
|
value: 89.554 |
|
- type: recall_at_100 |
|
value: 98.914 |
|
- type: recall_at_1000 |
|
value: 99.754 |
|
- type: recall_at_3 |
|
value: 72.89999999999999 |
|
- type: recall_at_5 |
|
value: 82.1 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/quora |
|
name: MTEB QuoraRetrieval |
|
config: default |
|
split: test |
|
revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 |
|
metrics: |
|
- type: map_at_1 |
|
value: 72.748 |
|
- type: map_at_10 |
|
value: 86.87700000000001 |
|
- type: map_at_100 |
|
value: 87.46199999999999 |
|
- type: map_at_1000 |
|
value: 87.47399999999999 |
|
- type: map_at_3 |
|
value: 83.95700000000001 |
|
- type: map_at_5 |
|
value: 85.82300000000001 |
|
- type: mrr_at_1 |
|
value: 83.62 |
|
- type: mrr_at_10 |
|
value: 89.415 |
|
- type: mrr_at_100 |
|
value: 89.484 |
|
- type: mrr_at_1000 |
|
value: 89.484 |
|
- type: mrr_at_3 |
|
value: 88.633 |
|
- type: mrr_at_5 |
|
value: 89.176 |
|
- type: ndcg_at_1 |
|
value: 83.62 |
|
- type: ndcg_at_10 |
|
value: 90.27 |
|
- type: ndcg_at_100 |
|
value: 91.23599999999999 |
|
- type: ndcg_at_1000 |
|
value: 91.293 |
|
- type: ndcg_at_3 |
|
value: 87.69500000000001 |
|
- type: ndcg_at_5 |
|
value: 89.171 |
|
- type: precision_at_1 |
|
value: 83.62 |
|
- type: precision_at_10 |
|
value: 13.683 |
|
- type: precision_at_100 |
|
value: 1.542 |
|
- type: precision_at_1000 |
|
value: 0.157 |
|
- type: precision_at_3 |
|
value: 38.363 |
|
- type: precision_at_5 |
|
value: 25.196 |
|
- type: recall_at_1 |
|
value: 72.748 |
|
- type: recall_at_10 |
|
value: 96.61699999999999 |
|
- type: recall_at_100 |
|
value: 99.789 |
|
- type: recall_at_1000 |
|
value: 99.997 |
|
- type: recall_at_3 |
|
value: 89.21 |
|
- type: recall_at_5 |
|
value: 93.418 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/reddit-clustering |
|
name: MTEB RedditClustering |
|
config: default |
|
split: test |
|
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb |
|
metrics: |
|
- type: v_measure |
|
value: 61.51909029379199 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/reddit-clustering-p2p |
|
name: MTEB RedditClusteringP2P |
|
config: default |
|
split: test |
|
revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 |
|
metrics: |
|
- type: v_measure |
|
value: 68.24483162045645 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/scidocs |
|
name: MTEB SCIDOCS |
|
config: default |
|
split: test |
|
revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 |
|
metrics: |
|
- type: map_at_1 |
|
value: 4.793 |
|
- type: map_at_10 |
|
value: 13.092 |
|
- type: map_at_100 |
|
value: 15.434000000000001 |
|
- type: map_at_1000 |
|
value: 15.748999999999999 |
|
- type: map_at_3 |
|
value: 9.139 |
|
- type: map_at_5 |
|
value: 11.033 |
|
- type: mrr_at_1 |
|
value: 23.599999999999998 |
|
- type: mrr_at_10 |
|
value: 35.892 |
|
- type: mrr_at_100 |
|
value: 36.962 |
|
- type: mrr_at_1000 |
|
value: 37.009 |
|
- type: mrr_at_3 |
|
value: 32.550000000000004 |
|
- type: mrr_at_5 |
|
value: 34.415 |
|
- type: ndcg_at_1 |
|
value: 23.599999999999998 |
|
- type: ndcg_at_10 |
|
value: 21.932 |
|
- type: ndcg_at_100 |
|
value: 30.433 |
|
- type: ndcg_at_1000 |
|
value: 35.668 |
|
- type: ndcg_at_3 |
|
value: 20.483999999999998 |
|
- type: ndcg_at_5 |
|
value: 17.964 |
|
- type: precision_at_1 |
|
value: 23.599999999999998 |
|
- type: precision_at_10 |
|
value: 11.63 |
|
- type: precision_at_100 |
|
value: 2.383 |
|
- type: precision_at_1000 |
|
value: 0.363 |
|
- type: precision_at_3 |
|
value: 19.567 |
|
- type: precision_at_5 |
|
value: 16.06 |
|
- type: recall_at_1 |
|
value: 4.793 |
|
- type: recall_at_10 |
|
value: 23.558 |
|
- type: recall_at_100 |
|
value: 48.376999999999995 |
|
- type: recall_at_1000 |
|
value: 73.75699999999999 |
|
- type: recall_at_3 |
|
value: 11.903 |
|
- type: recall_at_5 |
|
value: 16.278000000000002 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sickr-sts |
|
name: MTEB SICK-R |
|
config: default |
|
split: test |
|
revision: 20a6d6f312dd54037fe07a32d58e5e168867909d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 87.31937967632581 |
|
- type: cos_sim_spearman |
|
value: 84.30523596401186 |
|
- type: euclidean_pearson |
|
value: 84.19537987069458 |
|
- type: euclidean_spearman |
|
value: 84.30522052876 |
|
- type: manhattan_pearson |
|
value: 84.16420807244911 |
|
- type: manhattan_spearman |
|
value: 84.28515410219309 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts12-sts |
|
name: MTEB STS12 |
|
config: default |
|
split: test |
|
revision: a0d554a64d88156834ff5ae9920b964011b16384 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 86.17180810119646 |
|
- type: cos_sim_spearman |
|
value: 78.44413657529002 |
|
- type: euclidean_pearson |
|
value: 81.69054139101816 |
|
- type: euclidean_spearman |
|
value: 78.44412412142488 |
|
- type: manhattan_pearson |
|
value: 82.04975789626462 |
|
- type: manhattan_spearman |
|
value: 78.78390856857253 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts13-sts |
|
name: MTEB STS13 |
|
config: default |
|
split: test |
|
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 88.35737871089687 |
|
- type: cos_sim_spearman |
|
value: 88.26850223126127 |
|
- type: euclidean_pearson |
|
value: 87.44100858335746 |
|
- type: euclidean_spearman |
|
value: 88.26850223126127 |
|
- type: manhattan_pearson |
|
value: 87.61572015772133 |
|
- type: manhattan_spearman |
|
value: 88.56229552813319 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts14-sts |
|
name: MTEB STS14 |
|
config: default |
|
split: test |
|
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 86.8395966764906 |
|
- type: cos_sim_spearman |
|
value: 84.49441798385489 |
|
- type: euclidean_pearson |
|
value: 85.3259176121388 |
|
- type: euclidean_spearman |
|
value: 84.49442124804686 |
|
- type: manhattan_pearson |
|
value: 85.35153862806513 |
|
- type: manhattan_spearman |
|
value: 84.60094577432503 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts15-sts |
|
name: MTEB STS15 |
|
config: default |
|
split: test |
|
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 90.14048269057345 |
|
- type: cos_sim_spearman |
|
value: 90.27866978947013 |
|
- type: euclidean_pearson |
|
value: 89.35308361940393 |
|
- type: euclidean_spearman |
|
value: 90.27866978947013 |
|
- type: manhattan_pearson |
|
value: 89.37601244066997 |
|
- type: manhattan_spearman |
|
value: 90.42707449698062 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts16-sts |
|
name: MTEB STS16 |
|
config: default |
|
split: test |
|
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 86.8522678865688 |
|
- type: cos_sim_spearman |
|
value: 87.37396401580446 |
|
- type: euclidean_pearson |
|
value: 86.37219665505377 |
|
- type: euclidean_spearman |
|
value: 87.37396385867791 |
|
- type: manhattan_pearson |
|
value: 86.44628823799896 |
|
- type: manhattan_spearman |
|
value: 87.49116026788859 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (en-en) |
|
config: en-en |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 92.94248481968916 |
|
- type: cos_sim_spearman |
|
value: 92.68185242943188 |
|
- type: euclidean_pearson |
|
value: 92.33802342092979 |
|
- type: euclidean_spearman |
|
value: 92.68185242943188 |
|
- type: manhattan_pearson |
|
value: 92.2011323340474 |
|
- type: manhattan_spearman |
|
value: 92.43364757640346 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (en) |
|
config: en |
|
split: test |
|
revision: eea2b4fe26a775864c896887d910b76a8098ad3f |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 70.2918782293091 |
|
- type: cos_sim_spearman |
|
value: 68.61986257003369 |
|
- type: euclidean_pearson |
|
value: 70.51920905899138 |
|
- type: euclidean_spearman |
|
value: 68.61986257003369 |
|
- type: manhattan_pearson |
|
value: 70.64673843811433 |
|
- type: manhattan_spearman |
|
value: 68.86711466517345 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/stsbenchmark-sts |
|
name: MTEB STSBenchmark |
|
config: default |
|
split: test |
|
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 88.62956838105524 |
|
- type: cos_sim_spearman |
|
value: 88.80650007123052 |
|
- type: euclidean_pearson |
|
value: 88.37976252122822 |
|
- type: euclidean_spearman |
|
value: 88.80650007123052 |
|
- type: manhattan_pearson |
|
value: 88.49866938476616 |
|
- type: manhattan_spearman |
|
value: 89.02489665452616 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/scidocs-reranking |
|
name: MTEB SciDocsRR |
|
config: default |
|
split: test |
|
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab |
|
metrics: |
|
- type: map |
|
value: 86.40175229911527 |
|
- type: mrr |
|
value: 96.61958230585682 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/scifact |
|
name: MTEB SciFact |
|
config: default |
|
split: test |
|
revision: 0228b52cf27578f30900b9e5271d331663a030d7 |
|
metrics: |
|
- type: map_at_1 |
|
value: 63.05 |
|
- type: map_at_10 |
|
value: 73.844 |
|
- type: map_at_100 |
|
value: 74.313 |
|
- type: map_at_1000 |
|
value: 74.321 |
|
- type: map_at_3 |
|
value: 71.17999999999999 |
|
- type: map_at_5 |
|
value: 72.842 |
|
- type: mrr_at_1 |
|
value: 65.667 |
|
- type: mrr_at_10 |
|
value: 74.772 |
|
- type: mrr_at_100 |
|
value: 75.087 |
|
- type: mrr_at_1000 |
|
value: 75.095 |
|
- type: mrr_at_3 |
|
value: 72.944 |
|
- type: mrr_at_5 |
|
value: 74.078 |
|
- type: ndcg_at_1 |
|
value: 65.667 |
|
- type: ndcg_at_10 |
|
value: 78.31700000000001 |
|
- type: ndcg_at_100 |
|
value: 79.969 |
|
- type: ndcg_at_1000 |
|
value: 80.25 |
|
- type: ndcg_at_3 |
|
value: 74.099 |
|
- type: ndcg_at_5 |
|
value: 76.338 |
|
- type: precision_at_1 |
|
value: 65.667 |
|
- type: precision_at_10 |
|
value: 10.233 |
|
- type: precision_at_100 |
|
value: 1.107 |
|
- type: precision_at_1000 |
|
value: 0.11299999999999999 |
|
- type: precision_at_3 |
|
value: 28.889 |
|
- type: precision_at_5 |
|
value: 19.0 |
|
- type: recall_at_1 |
|
value: 63.05 |
|
- type: recall_at_10 |
|
value: 90.822 |
|
- type: recall_at_100 |
|
value: 97.667 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_3 |
|
value: 79.489 |
|
- type: recall_at_5 |
|
value: 85.161 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/sprintduplicatequestions-pairclassification |
|
name: MTEB SprintDuplicateQuestions |
|
config: default |
|
split: test |
|
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 99.83564356435643 |
|
- type: cos_sim_ap |
|
value: 96.10619363017767 |
|
- type: cos_sim_f1 |
|
value: 91.61225514816677 |
|
- type: cos_sim_precision |
|
value: 92.02825428859738 |
|
- type: cos_sim_recall |
|
value: 91.2 |
|
- type: dot_accuracy |
|
value: 99.83564356435643 |
|
- type: dot_ap |
|
value: 96.10619363017767 |
|
- type: dot_f1 |
|
value: 91.61225514816677 |
|
- type: dot_precision |
|
value: 92.02825428859738 |
|
- type: dot_recall |
|
value: 91.2 |
|
- type: euclidean_accuracy |
|
value: 99.83564356435643 |
|
- type: euclidean_ap |
|
value: 96.10619363017769 |
|
- type: euclidean_f1 |
|
value: 91.61225514816677 |
|
- type: euclidean_precision |
|
value: 92.02825428859738 |
|
- type: euclidean_recall |
|
value: 91.2 |
|
- type: manhattan_accuracy |
|
value: 99.84158415841584 |
|
- type: manhattan_ap |
|
value: 96.27527798658713 |
|
- type: manhattan_f1 |
|
value: 92.0 |
|
- type: manhattan_precision |
|
value: 92.0 |
|
- type: manhattan_recall |
|
value: 92.0 |
|
- type: max_accuracy |
|
value: 99.84158415841584 |
|
- type: max_ap |
|
value: 96.27527798658713 |
|
- type: max_f1 |
|
value: 92.0 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/stackexchange-clustering |
|
name: MTEB StackExchangeClustering |
|
config: default |
|
split: test |
|
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 |
|
metrics: |
|
- type: v_measure |
|
value: 76.93753872885304 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/stackexchange-clustering-p2p |
|
name: MTEB StackExchangeClusteringP2P |
|
config: default |
|
split: test |
|
revision: 815ca46b2622cec33ccafc3735d572c266efdb44 |
|
metrics: |
|
- type: v_measure |
|
value: 46.044085080870126 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/stackoverflowdupquestions-reranking |
|
name: MTEB StackOverflowDupQuestions |
|
config: default |
|
split: test |
|
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 |
|
metrics: |
|
- type: map |
|
value: 55.885129730227256 |
|
- type: mrr |
|
value: 56.95062494694848 |
|
- task: |
|
type: Summarization |
|
dataset: |
|
type: mteb/summeval |
|
name: MTEB SummEval |
|
config: default |
|
split: test |
|
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 31.202047940935508 |
|
- type: cos_sim_spearman |
|
value: 30.984832035722228 |
|
- type: dot_pearson |
|
value: 31.20204247226978 |
|
- type: dot_spearman |
|
value: 30.984832035722228 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/trec-covid |
|
name: MTEB TRECCOVID |
|
config: default |
|
split: test |
|
revision: bb9466bac8153a0349341eb1b22e06409e78ef4e |
|
metrics: |
|
- type: map_at_1 |
|
value: 0.245 |
|
- type: map_at_10 |
|
value: 2.249 |
|
- type: map_at_100 |
|
value: 14.85 |
|
- type: map_at_1000 |
|
value: 36.596000000000004 |
|
- type: map_at_3 |
|
value: 0.717 |
|
- type: map_at_5 |
|
value: 1.18 |
|
- type: mrr_at_1 |
|
value: 94.0 |
|
- type: mrr_at_10 |
|
value: 96.167 |
|
- type: mrr_at_100 |
|
value: 96.167 |
|
- type: mrr_at_1000 |
|
value: 96.167 |
|
- type: mrr_at_3 |
|
value: 95.667 |
|
- type: mrr_at_5 |
|
value: 96.167 |
|
- type: ndcg_at_1 |
|
value: 91.0 |
|
- type: ndcg_at_10 |
|
value: 87.09700000000001 |
|
- type: ndcg_at_100 |
|
value: 69.637 |
|
- type: ndcg_at_1000 |
|
value: 62.257 |
|
- type: ndcg_at_3 |
|
value: 90.235 |
|
- type: ndcg_at_5 |
|
value: 89.51400000000001 |
|
- type: precision_at_1 |
|
value: 94.0 |
|
- type: precision_at_10 |
|
value: 90.60000000000001 |
|
- type: precision_at_100 |
|
value: 71.38 |
|
- type: precision_at_1000 |
|
value: 27.400000000000002 |
|
- type: precision_at_3 |
|
value: 94.0 |
|
- type: precision_at_5 |
|
value: 93.2 |
|
- type: recall_at_1 |
|
value: 0.245 |
|
- type: recall_at_10 |
|
value: 2.366 |
|
- type: recall_at_100 |
|
value: 17.491 |
|
- type: recall_at_1000 |
|
value: 58.772999999999996 |
|
- type: recall_at_3 |
|
value: 0.7270000000000001 |
|
- type: recall_at_5 |
|
value: 1.221 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: mteb/touche2020 |
|
name: MTEB Touche2020 |
|
config: default |
|
split: test |
|
revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f |
|
metrics: |
|
- type: map_at_1 |
|
value: 3.435 |
|
- type: map_at_10 |
|
value: 12.147 |
|
- type: map_at_100 |
|
value: 18.724 |
|
- type: map_at_1000 |
|
value: 20.426 |
|
- type: map_at_3 |
|
value: 6.526999999999999 |
|
- type: map_at_5 |
|
value: 9.198 |
|
- type: mrr_at_1 |
|
value: 48.980000000000004 |
|
- type: mrr_at_10 |
|
value: 62.970000000000006 |
|
- type: mrr_at_100 |
|
value: 63.288999999999994 |
|
- type: mrr_at_1000 |
|
value: 63.288999999999994 |
|
- type: mrr_at_3 |
|
value: 59.184000000000005 |
|
- type: mrr_at_5 |
|
value: 61.224000000000004 |
|
- type: ndcg_at_1 |
|
value: 46.939 |
|
- type: ndcg_at_10 |
|
value: 30.61 |
|
- type: ndcg_at_100 |
|
value: 41.683 |
|
- type: ndcg_at_1000 |
|
value: 53.144000000000005 |
|
- type: ndcg_at_3 |
|
value: 36.284 |
|
- type: ndcg_at_5 |
|
value: 34.345 |
|
- type: precision_at_1 |
|
value: 48.980000000000004 |
|
- type: precision_at_10 |
|
value: 26.122 |
|
- type: precision_at_100 |
|
value: 8.204 |
|
- type: precision_at_1000 |
|
value: 1.6019999999999999 |
|
- type: precision_at_3 |
|
value: 35.374 |
|
- type: precision_at_5 |
|
value: 32.653 |
|
- type: recall_at_1 |
|
value: 3.435 |
|
- type: recall_at_10 |
|
value: 18.953 |
|
- type: recall_at_100 |
|
value: 50.775000000000006 |
|
- type: recall_at_1000 |
|
value: 85.858 |
|
- type: recall_at_3 |
|
value: 7.813000000000001 |
|
- type: recall_at_5 |
|
value: 11.952 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/toxic_conversations_50k |
|
name: MTEB ToxicConversationsClassification |
|
config: default |
|
split: test |
|
revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de |
|
metrics: |
|
- type: accuracy |
|
value: 71.2938 |
|
- type: ap |
|
value: 15.090139095602268 |
|
- type: f1 |
|
value: 55.23862650598296 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/tweet_sentiment_extraction |
|
name: MTEB TweetSentimentExtractionClassification |
|
config: default |
|
split: test |
|
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a |
|
metrics: |
|
- type: accuracy |
|
value: 64.7623089983022 |
|
- type: f1 |
|
value: 65.07617131099336 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/twentynewsgroups-clustering |
|
name: MTEB TwentyNewsgroupsClustering |
|
config: default |
|
split: test |
|
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 |
|
metrics: |
|
- type: v_measure |
|
value: 57.2988222684939 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/twittersemeval2015-pairclassification |
|
name: MTEB TwitterSemEval2015 |
|
config: default |
|
split: test |
|
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 88.6034451928235 |
|
- type: cos_sim_ap |
|
value: 81.51815279166863 |
|
- type: cos_sim_f1 |
|
value: 74.43794671864849 |
|
- type: cos_sim_precision |
|
value: 73.34186939820742 |
|
- type: cos_sim_recall |
|
value: 75.56728232189973 |
|
- type: dot_accuracy |
|
value: 88.6034451928235 |
|
- type: dot_ap |
|
value: 81.51816956866841 |
|
- type: dot_f1 |
|
value: 74.43794671864849 |
|
- type: dot_precision |
|
value: 73.34186939820742 |
|
- type: dot_recall |
|
value: 75.56728232189973 |
|
- type: euclidean_accuracy |
|
value: 88.6034451928235 |
|
- type: euclidean_ap |
|
value: 81.51817015121485 |
|
- type: euclidean_f1 |
|
value: 74.43794671864849 |
|
- type: euclidean_precision |
|
value: 73.34186939820742 |
|
- type: euclidean_recall |
|
value: 75.56728232189973 |
|
- type: manhattan_accuracy |
|
value: 88.5736424867378 |
|
- type: manhattan_ap |
|
value: 81.37610101292196 |
|
- type: manhattan_f1 |
|
value: 74.2504182215931 |
|
- type: manhattan_precision |
|
value: 72.46922883697563 |
|
- type: manhattan_recall |
|
value: 76.12137203166228 |
|
- type: max_accuracy |
|
value: 88.6034451928235 |
|
- type: max_ap |
|
value: 81.51817015121485 |
|
- type: max_f1 |
|
value: 74.43794671864849 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/twitterurlcorpus-pairclassification |
|
name: MTEB TwitterURLCorpus |
|
config: default |
|
split: test |
|
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 89.53118329646446 |
|
- type: cos_sim_ap |
|
value: 87.41972033060013 |
|
- type: cos_sim_f1 |
|
value: 79.4392523364486 |
|
- type: cos_sim_precision |
|
value: 75.53457372951958 |
|
- type: cos_sim_recall |
|
value: 83.7696335078534 |
|
- type: dot_accuracy |
|
value: 89.53118329646446 |
|
- type: dot_ap |
|
value: 87.41971646088945 |
|
- type: dot_f1 |
|
value: 79.4392523364486 |
|
- type: dot_precision |
|
value: 75.53457372951958 |
|
- type: dot_recall |
|
value: 83.7696335078534 |
|
- type: euclidean_accuracy |
|
value: 89.53118329646446 |
|
- type: euclidean_ap |
|
value: 87.41972415605997 |
|
- type: euclidean_f1 |
|
value: 79.4392523364486 |
|
- type: euclidean_precision |
|
value: 75.53457372951958 |
|
- type: euclidean_recall |
|
value: 83.7696335078534 |
|
- type: manhattan_accuracy |
|
value: 89.5855163581325 |
|
- type: manhattan_ap |
|
value: 87.51158697451964 |
|
- type: manhattan_f1 |
|
value: 79.54455087655883 |
|
- type: manhattan_precision |
|
value: 74.96763643796416 |
|
- type: manhattan_recall |
|
value: 84.71666153372344 |
|
- type: max_accuracy |
|
value: 89.5855163581325 |
|
- type: max_ap |
|
value: 87.51158697451964 |
|
- type: max_f1 |
|
value: 79.54455087655883 |
|
language: |
|
- en |
|
license: cc-by-nc-4.0 |
|
--- |
|
<h1 align="center">Linq-AI-Research/Linq-Embed-Mistral</h1> |
|
|
|
**Linq-Embed-Mistral** |
|
|
|
Linq-Embed-Mistral has been developed by building upon the foundations of the [E5-mistral-7b-instruct](https://huggingface.co/intfloat/e5-mistral-7b-instruct) and [Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) models. We focus on improving text retrieval using advanced data refinement methods, including sophisticated data crafting, data filtering, and negative mining guided by teacher models, which are highly tailored to each task, to improve the quality of the synthetic data generated by LLM. These methods are applied to both existing benchmark dataset and highly tailored synthetic dataset generated via LLMs. Our efforts primarily aim to create high-quality triplet datasets (query, positive example, negative example), significantly improving text retrieval performance. |
|
|
|
Linq-Embed-Mistral performs well in the MTEB benchmarks (as of May 29, 2024). The model excels in retrieval tasks, ranking <ins>**`1st`**</ins> among all models listed on the MTEB leaderboard with a performance score of <ins>**`60.2`**</ins>. This outstanding performance underscores its superior capability in enhancing search precision and reliability. The model achieves an average score of <ins>**`68.2`**</ins> across 56 datasets in the MTEB benchmarks, making it the highest-ranking publicly accessible model and third overall. (Please note that [NV-Emb-v1](https://huggingface.co/nvidia/NV-Embed-v1) and [voyage-large-2-instruct](https://docs.voyageai.com/embeddings/), ranked 1st and 2nd on the leaderboard as of May 29, reported their performance without releasing their models.) |
|
|
|
|
|
This project is for research purposes only. Third-party datasets may be subject to additional terms and conditions under their associated licenses. Please refer to specific papers for more details: |
|
|
|
- [MTEB benchmark](https://arxiv.org/abs/2210.07316) |
|
- [Mistral](https://arxiv.org/abs/2310.06825) |
|
- [E5-mistral-7b-instruct](https://arxiv.org/pdf/2401.00368.pdf) |
|
|
|
For more details, refer to [this blog post](https://getlinq.com/blog/linq-embed-mistral/) and [this report](https://huggingface.co/Linq-AI-Research/Linq-Embed-Mistral/blob/main/LinqAIResearch2024_Linq-Embed-Mistral.pdf). |
|
|
|
## How to use |
|
|
|
Here is an example of how to encode queries and passages from the Mr.TyDi training dataset, both with Sentence Transformers or Transformers directly. |
|
|
|
### Sentence Transformers |
|
|
|
```python |
|
from sentence_transformers import SentenceTransformer |
|
|
|
# Load the model |
|
model = SentenceTransformer("Linq-AI-Research/Linq-Embed-Mistral") |
|
|
|
# Each query must come with a one-sentence instruction that describes the task |
|
task = 'Given a question, retrieve Wikipedia passages that answer the question' |
|
prompt = f"Instruct: {task}\nQuery: " |
|
queries = [ |
|
"최초의 원자력 발전소는 무엇인가?", |
|
"Who invented Hangul?" |
|
] |
|
passages = [ |
|
"현재 사용되는 핵분열 방식을 이용한 전력생산은 1948년 9월 미국 테네시주 오크리지에 설치된 X-10 흑연원자로에서 전구의 불을 밝히는 데 사용되면서 시작되었다. 그리고 1954년 6월에 구소련의 오브닌스크에 건설된 흑연감속 비등경수 압력관형 원자로를 사용한 오브닌스크 원자력 발전소가 시험적으로 전력생산을 시작하였고, 최초의 상업용 원자력 엉더이로를 사용한 영국 셀라필드 원자력 단지에 위치한 콜더 홀(Calder Hall) 원자력 발전소로, 1956년 10월 17일 상업 운전을 시작하였다.", |
|
"Hangul was personally created and promulgated by the fourth king of the Joseon dynasty, Sejong the Great.[1][2] Sejong's scholarly institute, the Hall of Worthies, is often credited with the work, and at least one of its scholars was heavily involved in its creation, but it appears to have also been a personal project of Sejong." |
|
] |
|
|
|
# Encode the queries and passages. We only use the prompt for the queries |
|
query_embeddings = model.encode(queries, prompt=prompt) |
|
passage_embeddings = model.encode(passages) |
|
|
|
# Compute the (cosine) similarity scores |
|
scores = model.similarity(query_embeddings, passage_embeddings) * 100 |
|
print(scores.tolist()) |
|
# [[73.72908782958984, 30.122787475585938], [29.15508460998535, 79.25375366210938]] |
|
``` |
|
|
|
### Transformers |
|
|
|
```python |
|
import torch |
|
import torch.nn.functional as F |
|
from torch import Tensor |
|
from transformers import AutoTokenizer, AutoModel |
|
|
|
def last_token_pool(last_hidden_states: Tensor, |
|
attention_mask: Tensor) -> Tensor: |
|
left_padding = (attention_mask[:, -1].sum() == attention_mask.shape[0]) |
|
if left_padding: |
|
return last_hidden_states[:, -1] |
|
else: |
|
sequence_lengths = attention_mask.sum(dim=1) - 1 |
|
batch_size = last_hidden_states.shape[0] |
|
return last_hidden_states[torch.arange(batch_size, device=last_hidden_states.device), sequence_lengths] |
|
|
|
def get_detailed_instruct(task_description: str, query: str) -> str: |
|
return f'Instruct: {task_description}\nQuery: {query}' |
|
|
|
# Each query must come with a one-sentence instruction that describes the task |
|
task = 'Given a question, retrieve Wikipedia passages that answer the question' |
|
queries = [ |
|
get_detailed_instruct(task, '최초의 원자력 발전소는 무엇인가?'), |
|
get_detailed_instruct(task, 'Who invented Hangul?') |
|
] |
|
# No need to add instruction for retrieval documents |
|
passages = [ |
|
"현재 사용되는 핵분열 방식을 이용한 전력생산은 1948년 9월 미국 테네시주 오크리지에 설치된 X-10 흑연원자로에서 전구의 불을 밝히는 데 사용되면서 시작되었다. 그리고 1954년 6월에 구소련의 오브닌스크에 건설된 흑연감속 비등경수 압력관형 원자로를 사용한 오브닌스크 원자력 발전소가 시험적으로 전력생산을 시작하였고, 최초의 상업용 원자력 엉더이로를 사용한 영국 셀라필드 원자력 단지에 위치한 콜더 홀(Calder Hall) 원자력 발전소로, 1956년 10월 17일 상업 운전을 시작하였다.", |
|
"Hangul was personally created and promulgated by the fourth king of the Joseon dynasty, Sejong the Great.[1][2] Sejong's scholarly institute, the Hall of Worthies, is often credited with the work, and at least one of its scholars was heavily involved in its creation, but it appears to have also been a personal project of Sejong." |
|
] |
|
|
|
# Load model and tokenizer |
|
tokenizer = AutoTokenizer.from_pretrained('Linq-AI-Research/Linq-Embed-Mistral') |
|
model = AutoModel.from_pretrained('Linq-AI-Research/Linq-Embed-Mistral') |
|
|
|
max_length = 4096 |
|
input_texts = [*queries, *passages] |
|
# Tokenize the input texts |
|
batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors="pt") |
|
outputs = model(**batch_dict) |
|
embeddings = last_token_pool(outputs.last_hidden_state, batch_dict['attention_mask']) |
|
|
|
# Normalize embeddings |
|
embeddings = F.normalize(embeddings, p=2, dim=1) |
|
scores = (embeddings[:2] @ embeddings[2:].T) * 100 |
|
print(scores.tolist()) |
|
# [[73.72909545898438, 30.122783660888672], [29.155078887939453, 79.25374603271484]] |
|
``` |
|
|
|
### MTEB Benchmark Evaluation |
|
|
|
Check out [unilm/e5](https://github.com/microsoft/unilm/tree/master/e5) to reproduce evaluation results on the [BEIR](https://arxiv.org/abs/2104.08663) and [MTEB](https://arxiv.org/abs/2210.07316) benchmark. |
|
|
|
## Evaluation Result |
|
|
|
### MTEB (as of May 29, 2024) |
|
|
|
| Model Name | Retrieval (15) | Average (56) | |
|
| :------------------------------------------------------------------------------: | :------------: | :----------: | |
|
| [Linq-Embed-Mistral](https://huggingface.co/Linq-AI-Research/Linq-Embed-Mistral) | 60.2 | 68.2 | |
|
| [NV-Embed-v1](https://huggingface.co/nvidia/NV-Embed-v1) | 59.4 | 69.3 | |
|
| [SFR-Embedding-Mistral](https://huggingface.co/Salesforce/SFR-Embedding-Mistral) | 59.0 | 67.6 | |
|
| [voyage-large-2-instruct](https://docs.voyageai.com/docs/embeddings) | 58.3 | 68.3 | |
|
| [GritLM-7B](https://huggingface.co/GritLM/GritLM-7B) | 57.4 | 66.8 | |
|
| [voyage-lite-02-instruct](https://docs.voyageai.com/docs/embeddings) | 56.6 | 67.1 | |
|
|[gte-Qwen1.5-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen1.5-7B-instruct)| 56.2 | 67.3 | |
|
| [e5-mistral-7b-instruct](https://huggingface.co/intfloat/e5-mistral-7b-instruct) | 56.9 | 66.6 | |
|
|[google-gecko.text-embedding-preview-0409](https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings?hl=ko#latest_models)| 55.7 | 66.3 | |
|
|[text-embedding-3-large](https://openai.com/index/new-embedding-models-and-api-updates/)| 55.4 | 64.6 | |
|
|[Cohere-embed-english-v3.0](https://huggingface.co/Cohere/Cohere-embed-english-v3.0)| 55.0 | 64.5 | |
|
|
|
# Linq Research Team. |
|
|
|
- [Junseong Kim](https://huggingface.co/Junseong) |
|
- [Seolhwa Lee](https://huggingface.co/Seolhwa) |
|
- [Jihoon Kwon](https://huggingface.co/Mayfull) |
|
- [Sangmo Gu](https://huggingface.co/karma-os) |
|
- Yejin Kim |
|
- Minkyung Cho |
|
- [Jy-yong Sohn](https://itml.yonsei.ac.kr/professor) |
|
- [Chanyeol Choi](https://www.linkedin.com/in/chanyeolchoi) |
|
|
|
# Citation |
|
|
|
```bibtex |
|
@misc{LinqAIResearch2024, |
|
title={Linq-Embed-Mistral:Elevating Text Retrieval with Improved GPT Data Through Task-Specific Control and Quality Refinement}, |
|
author={Junseong Kim, Seolhwa Lee, Jihoon Kwon, Sangmo Gu, Yejin Kim, Minkyung Cho, Jy-yong Sohn, Chanyeol Choi}, |
|
howpublished={Linq AI Research Blog}, |
|
year={2024}, |
|
url={https://getlinq.com/blog/linq-embed-mistral/} |
|
} |
|
``` |
|
|
