wip adding symbols to model types

app.py

@@ -179,6 +179,7 @@ def add_new_eval(
     precision: str,
     private: bool,
     weight_type: str,
+    model_type: str,
 ):
     precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -209,6 +210,7 @@ def add_new_eval(
         "weight_type": weight_type,
         "status": "PENDING",
         "submitted_time": current_time,
+        "model_type": model_type,
     }
 
     user_name = ""
@@ -396,6 +398,14 @@ with demo:
                         max_choices=1,
                         interactive=True,
                     )
+                    model_type = gr.Dropdown(
+                        choices=["pretrained", "fine-tuned", "with RL"], 
+                        label="Model type", 
+                        multiselect=False,
+                        value="pretrained",
+                        max_choices=1,
+                        interactive=True,
+                    )
                     weight_type = gr.Dropdown(
                         choices=["Original", "Delta", "Adapter"],
                         label="Weights type", 
@@ -419,6 +429,7 @@ with demo:
                     precision,
                     private,
                     weight_type,
+                    model_type
                 ],
                 submission_result,
             )

src/assets/text_content.py

@@ -75,6 +75,7 @@ With the plethora of large language models (LLMs) and chatbots being released we
 - <a href="https://arxiv.org/abs/2009.03300" target="_blank">  MMLU </a>  (5-shot) - a test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.
 - <a href="https://arxiv.org/abs/2109.07958" target="_blank">  TruthfulQA </a> (0-shot) - a test to measure a model’s propensity to reproduce falsehoods commonly found online. Note: TruthfulQA in the Harness is actually a minima a 6-shots task, as it is prepended by 6 examples systematically, even when launched using 0 for the number of few-shot examples.
 
+For all these evaluations, a higher score is a better score. 
 We chose these benchmarks as they test a variety of reasoning and general knowledge across a wide variety of fields in 0-shot and few-shot settings.
 
 # Some good practices before submitting a model
@@ -140,13 +141,13 @@ These models will be automatically evaluated on the 🤗 cluster.
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""@misc{open-llm-leaderboard,
+CITATION_BUTTON_TEXT = r"""
+@misc{open-llm-leaderboard,
   author = {Edward Beeching, Clémentine Fourrier, Nathan Habib, Sheon Han, Nathan Lambert, Nazneen Rajani, Omar Sanseviero, Lewis Tunstall, Thomas Wolf},
   title = {Open LLM Leaderboard},
   year = {2023},
   publisher = {Hugging Face},
   howpublished = "\url{https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard}"
-
 }
 @software{eval-harness,
   author       = {Gao, Leo and

src/auto_leaderboard/model_metadata_type.py

@@ -1,10 +1,17 @@
+from dataclasses import dataclass
 from enum import Enum
 from typing import Dict, List
 
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str # emoji
+
+
 class ModelType(Enum):
-    PT = "pretrained"
-    SFT = "finetuned"
-    RL = "with RL"
+    PT = ModelInfo(name="pretrained", symbol="🟢")
+    SFT = ModelInfo(name="finetuned", symbol="🔶")
+    RL = ModelInfo(name="with RL", symbol="🟦")
 
 
 TYPE_METADATA: Dict[str, ModelType] = {
@@ -160,13 +167,23 @@ TYPE_METADATA: Dict[str, ModelType] = {
 
 def get_model_type(leaderboard_data: List[dict]):
     for model_data in leaderboard_data:
-        model_data["Type"] = TYPE_METADATA.get(model_data["model_name_for_query"], "N/A")
-        if model_data["Type"] == "N/A":
+        # Init
+        model_data["Type name"] = "N/A"
+        model_data["Type"] = ""
+
+        # Stored information
+        if model_data["model_name_for_query"] in TYPE_METADATA:
+            model_data["Type name"] = TYPE_METADATA[model_data["model_name_for_query"]].value.name
+            model_data["Type"] = TYPE_METADATA[model_data["model_name_for_query"]].value.symbol
+        else: # Supposed from the name
             if any([i in model_data["model_name_for_query"] for i in ["finetuned", "-ft-"]]):
-                model_data["Type"] = ModelType.SFT
+                model_data["Type name"] = ModelType.SFT.value.name
+                model_data["Type"] = ModelType.SFT.value.symbol
             elif any([i in model_data["model_name_for_query"] for i in ["pretrained"]]):
-                model_data["Type"] = ModelType.PT
+                model_data["Type name"] = ModelType.PT.value.name
+                model_data["Type"] = ModelType.PT.value.symbol
             elif any([i in model_data["model_name_for_query"] for i in ["-rl-", "-rlhf-"]]):
-                model_data["Type"] = ModelType.RL
+                model_data["Type name"] = ModelType.RL.value.name
+                model_data["Type"] = ModelType.RL.value.symbol
  
  

src/utils_display.py

@@ -14,13 +14,14 @@ def fields(raw_class):
 
 @dataclass(frozen=True)
 class AutoEvalColumn: # Auto evals column
+    model_type_symbol = ColumnContent("Type", "str", True)
     model = ColumnContent("Model", "markdown", True)
     average = ColumnContent("Average ⬆️", "number", True)
-    arc = ColumnContent("ARC ⬆️", "number", True)
-    hellaswag = ColumnContent("HellaSwag ⬆️", "number", True)
-    mmlu = ColumnContent("MMLU ⬆️", "number", True)
+    arc = ColumnContent("ARC", "number", True)
+    hellaswag = ColumnContent("HellaSwag", "number", True)
+    mmlu = ColumnContent("MMLU", "number", True)
     truthfulqa = ColumnContent("TruthfulQA (MC) ⬆️", "number", True)
-    model_type = ColumnContent("Type", "str", False)
+    model_type = ColumnContent("Type name", "str", False)
     precision = ColumnContent("Precision", "str", False, True)
     license = ColumnContent("Hub License", "str", False)
     params = ColumnContent("#Params (B)", "number", False)