init commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,13 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -0,0 +1,45 @@
+---
+title: Leaderboard
+emoji: 🥇
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+sdk_version: 4.26.0
+app_file: app.py
+pinned: true
+license: apache-2.0
+---
+
+# Start the configuration
+
+Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
+
+Results files should have the following format and be stored as json files:
+```json
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.
+
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+
+# Code logic for more complex edits
+
+You'll find 
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -0,0 +1,65 @@
+import subprocess
+import gradio as gr
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+
+from src.pages.about import show_about_page
+from src.pages.submit import show_submit_page
+from src.pages.result_table import show_result_page
+from src.about import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    INTRODUCTION_TEXT,
+    TITLE,
+)
+from src.display.css_html_js import custom_css
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+
+
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)
+
+try:
+    print(EVAL_REQUESTS_PATH)
+    snapshot_download(
+        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
+try:
+    print(EVAL_RESULTS_PATH)
+    snapshot_download(
+        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
+
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        show_result_page(root_path='MC', title='📝 Exam', index=0)
+        show_result_page(root_path='LLM', title='🤖 LLM as Judge', index=1)
+        show_result_page(root_path='NLU', title='🕵️ NLU', index=2)
+        show_result_page(root_path='NLG', title='🖊️ NLG', index=3)
+        show_about_page(index=4)
+        show_submit_page(index=5)
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=8,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+demo.queue(default_concurrency_limit=40).launch()

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,12 @@
+APScheduler==3.10.1
+black==23.11.0
+click==8.1.3
+gradio
+gradio_client
+huggingface-hub>=0.18.0
+numpy==1.24.2
+pandas==2.0.0
+requests==2.28.2
+tqdm==4.65.0
+transformers==4.35.2
+python-dotenv

src/about.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass
+from enum import Enum
+
+
+
+NUM_FEWSHOT = 0 # Change with your few shot
+# ---------------------------------------------------
+
+TITLE = """<h1>🇹🇭 Thai LLM Leaderboard</h1>"""
+
+
+# <a href="url"></a>
+
+INTRODUCTION_TEXT = """
+The Thai-LLM Leaderboard 🇹🇭 focused on standardizing evaluation methods for large language models (LLMs) in the Thai language based on <a href="https://github.com/SEACrowd">Seacrowd</a>,
+As part of an open community project, we welcome you to submit new evaluation tasks or models. 
+This leaderboard is developed in collaboration with <a href="https://www.scb10x.com">SCB 10X</a>, <a href="https://www.vistec.ac.th/">Vistec</a>, and <a href="https://aisingapore.org/">AI-Singapore</a>. 
+"""
+
+LLM_BENCHMARKS_TEXT = f"""
+Evaluations
+The leaderboard currently consists of the following benchmarks:
+- Exam
+  - <a href="https://huggingface.co/datasets/scb10x/thai_exam">ThaiExam</a>: ThaiExam is a Thai language benchmark based on examinations for high-school students and investment professionals in Thailand.
+  - <a href="https://arxiv.org/abs/2306.05179">M3Exam</a>: M3Exam is a novel benchmark sourced from real and official human exam questions for evaluating LLMsin a multilingual, multimodal, and multilevel context. Here is Thai subset of M3Exam.
+- LLM as Judge
+  - Thai MT-Bench: <a href="https://arxiv.org/abs/2306.05685">MT-Bench</a> inspired varient of LLM as judges specifically developed by Vistec for Thai language and cultural.
+- NLU
+  - <a href="https://huggingface.co/datasets/facebook/belebele">Belebele</a>: Belebele is a multiple-choice machine reading comprehension (MRC) dataset spanning 122 language variants. Here is Thai subset of Belebele.
+  - <a href="https://huggingface.co/datasets/facebook/xnli">XNLI</a>: XNLI is an evaluation corpus for language transfer and cross-lingual sentence classification in 15 languages. Here is Thai subset of XNLI.
+  - <a href="https://huggingface.co/datasets/cambridgeltl/xcopa">XCOPA</a>: XCOPA is a translation and reannotation of the English COPA to measuring commonsense across languages. Here is Thai subset of XCOPA.
+  - <a href="https://huggingface.co/datasets/pythainlp/wisesight_sentiment">Wisesight</a>: Wisesight sentiment analysis corpus is a social media messages in Thai language with sentiment label.
+- NLG
+  - <a href="https://huggingface.co/datasets/csebuetnlp/xlsum">XLSum</a>: XLSum is a comprehensive and diverse dataset comprising 1.35 million professionally annotated article-summary pairs from BBC. Here is Thai subset of XLSum.
+  - <a href="https://huggingface.co/datasets/SEACrowd/flores200">Flores200</a>: FLORES is a benchmark dataset for machine translation between English and low-resource languages. Here is Thai subset of Flores200.
+  - <a href="https://huggingface.co/datasets/iapp/iapp_wiki_qa_squad">iapp Wiki QA Squad</a>: iapp Wiki QA Squad is an extractive question answering dataset from Thai Wikipedia articles.
+
+
+Metrics Implementations
+- BLEU is calculated using flores200 tokenizer using huggingface evaluate <a href="https://huggingface.co/spaces/evaluate-metric/sacrebleu">implementation</a>.
+- ROUGEL is calculated using pythainlp newmm tokenizer using huggingface evaluate <a href="https://huggingface.co/spaces/evaluate-metric/rouge">implementation</a>.
+- LLM as judge rating is judged by OpenAI gpt-4o-2024-05-13 using prompt specific by <a href="https://github.com/lm-sys/FastChat/blob/main/fastchat/llm_judge/data/judge_prompts.jsonl">lmsys MT-Bench</a>.
+
+Reproducibility
+
+To learn more about the evaluation pipeline and reproduce our results, check out the repository <a href="https://github.com/SEACrowd/seacrowd-experiments">seacrowd-experiments</a>.
+
+Acknowledgements
+
+We're grateful to community members for task and model submitting. To contribute, see submit tab.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{lovenia2024seacrowdmultilingualmultimodaldata,
+      title={SEACrowd: A Multilingual Multimodal Data Hub and Benchmark Suite for Southeast Asian Languages}, 
+      author={Holy Lovenia and Rahmad Mahendra and Salsabil Maulana Akbar and Lester James V. Miranda and Jennifer Santoso and Elyanah Aco and Akhdan Fadhilah and Jonibek Mansurov and Joseph Marvin Imperial and Onno P. Kampman and Joel Ruben Antony Moniz and Muhammad Ravi Shulthan Habibi and Frederikus Hudi and Railey Montalan and Ryan Ignatius and Joanito Agili Lopo and William Nixon and Börje F. Karlsson and James Jaya and Ryandito Diandaru and Yuze Gao and Patrick Amadeus and Bin Wang and Jan Christian Blaise Cruz and Chenxi Whitehouse and Ivan Halim Parmonangan and Maria Khelli and Wenyu Zhang and Lucky Susanto and Reynard Adha Ryanda and Sonny Lazuardi Hermawan and Dan John Velasco and Muhammad Dehan Al Kautsar and Willy Fitra Hendria and Yasmin Moslem and Noah Flynn and Muhammad Farid Adilazuarda and Haochen Li and Johanes Lee and R. Damanhuri and Shuo Sun and Muhammad Reza Qorib and Amirbek Djanibekov and Wei Qi Leong and Quyet V. Do and Niklas Muennighoff and Tanrada Pansuwan and Ilham Firdausi Putra and Yan Xu and Ngee Chia Tai and Ayu Purwarianti and Sebastian Ruder and William Tjhi and Peerat Limkonchotiwat and Alham Fikri Aji and Sedrick Keh and Genta Indra Winata and Ruochen Zhang and Fajri Koto and Zheng-Xin Yong and Samuel Cahyawijaya},
+      year={2024},
+      eprint={2406.10118},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2406.10118}, 
+}"""

src/display/css_html_js.py

@@ -0,0 +1,105 @@
+custom_css = """
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+#leaderboard-table {
+    margin-top: 15px
+}
+
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+}
+
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+
+#scale-logo .download {
+    display: none;
+}
+#filter_type{
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span{
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap{
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner{  
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input{
+    width: 1px
+}
+#filter-columns-type{
+    border:0;
+    padding:0.5;
+}
+#filter-columns-size{
+    border:0;
+    padding:0.5;
+}
+#box-filter > .form{
+    border: 0
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/display/formatting.py

@@ -0,0 +1,27 @@
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,61 @@
+from dataclasses import dataclass, make_dataclass
+import pandas as pd
+
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    status = ColumnContent("status", "str", True)
+
+## All the model information that we might need
+@dataclass
+class ModelDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = "" # emoji
+
+
+# Column selection
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+NUMERIC_INTERVALS = {
+    "?": pd.Interval(-1, 0, closed="right"),
+    "~1.5": pd.Interval(0, 2, closed="right"),
+    "~3": pd.Interval(2, 4, closed="right"),
+    "~7": pd.Interval(4, 9, closed="right"),
+    "~13": pd.Interval(9, 20, closed="right"),
+    "~35": pd.Interval(20, 45, closed="right"),
+    "~60": pd.Interval(45, 70, closed="right"),
+    "70+": pd.Interval(70, 10000, closed="right"),
+}

src/envs.py

@@ -0,0 +1,26 @@
+import os
+from dotenv import load_dotenv
+load_dotenv()
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("TOKEN") # A read/write token for your org
+
+OWNER = "kunato-lab" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
+
+REPO_ID = f"{OWNER}/leaderboard"
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/results"
+
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH=os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+
+API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -0,0 +1,219 @@
+from collections import OrderedDict
+import glob
+import json
+import os
+from dataclasses import dataclass
+import dateutil
+
+from src.submission.check_validity import get_model_size
+from src.envs import API
+from src.display.formatting import model_hyperlink
+from src.display.utils import AutoEvalColumn
+
+
+DATASET_TO_NAME_MAPPING = {
+    'wisesight_thai_sentiment_seacrowd_text': "Wisesight - SA",
+    'xnli.tha_seacrowd_pairs': "XNLI Thai - NLI",
+    "xcopa_tha_seacrowd_qa": "XCOPA Thai - Reasoning",
+    'belebele_tha_thai_seacrowd_qa': "Belebele - RC",
+    'xl_sum_tha_seacrowd_t2t': "XLSum Summarization",
+    'm3exam_tha_seacrowd_qa': 'M3Exam',
+    "flores200_eng_Latn_tha_Thai_seacrowd_t2t": 'Flores200 ENG->TH',
+    "flores200_tha_Thai_eng_Latn_seacrowd_t2t": 'Flores200 TH->ENG',
+    "iapp_squad_seacrowd_qa": "iapp Squad - QA",
+    "Writing": "MT-Bench: Writing",
+    'Math': "MT-Bench: Math",
+    'Coding': 'MT-Bench: Coding',
+    'Extraction': 'MT-Bench: Extraction',
+    'Reasoning': 'MT-Bench: Reasoning',
+    'Roleplay': 'MT-Bench: Roleplay',
+    'STEM': 'MT-Bench: STEM',
+    'Social Science': 'MT-Bench: Social Science',
+    'thaiexam_qa': "Thai Exam",
+
+    'lr_sum_tha_seacrowd_t2t': '',
+    'ntrex_128_eng-US_tha_seacrowd_t2t': '',
+    'ntrex_128_tha_eng-US_seacrowd_t2t': '',
+}
+
+METRICS_TO_NAME_MAPPING = {
+    'accuracy': "Acc",
+    'ROUGE1': '',
+    'ROUGE2': '',
+    'ROUGEL': 'ROUGEL',
+    'BLEU': '',
+    'SacreBLEU': 'BLEU',
+    'chrF++': '',
+    'avg_rating': "Rating"
+}
+
+def _parse_value_for_metric(value, metric):
+    if metric.lower() == 'accuracy':
+        return float("{:.4f}".format(value)) * 100
+    else:
+        return float("{:.2f}".format(value))
+
+def _get_model_size(model: str):
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision="main")
+        model_size = get_model_size(model_info=model_info)
+        return model_size
+    except Exception:
+        return 0
+
+FILTERED_ONLY_FIELD = ['params']
+
+
+@dataclass
+class EvalResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """
+    eval_name: str # org_model (uid) 
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    results: dict
+    params: str
+    model_link: str = ""
+    date: str = "" # submission date of request file
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}"
+        full_model = "/".join(org_and_model)
+        model_link = config.get('model_link', '')
+        params = config.get('params', '')
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for k in data['results'].keys():
+            results[k] = data['results'][k]
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            model_link=model_link,
+            org=org,
+            model=model,
+            results=results,
+            params=params,
+        )
+
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(f"Could not find request file for {self.org}/{self.model}")
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        data_dict = OrderedDict({
+            AutoEvalColumn.model.name: model_hyperlink(self.model_link, self.full_model),
+            'params': _get_model_size(self.model_link) if self.params == '' else self.params
+        })
+        for key in self.results.keys():
+            value = self.results[key]
+            for key2 in value.keys():
+                dataset_name_mapped = DATASET_TO_NAME_MAPPING[key]
+                metric_name_mapped = METRICS_TO_NAME_MAPPING[key2]
+                if dataset_name_mapped.strip() == '':
+                    continue
+                if metric_name_mapped.strip() == '':
+                    continue
+                data_dict[dataset_name_mapped + f'({metric_name_mapped})'] = _parse_value_for_metric(value[key2], key2)
+        # TODO add Average ⬆️
+        
+        avg = []
+        for key, value in data_dict.items():
+            if isinstance(value, float):
+                avg.append(data_dict[key])
+        data_dict['Average ⬆️'] = float("{:.2f}".format(sum(avg) / len(avg)))
+        for k in list(data_dict.keys()):
+            if k not in [AutoEvalColumn.model.name, 'Average ⬆️']:
+                data_dict.move_to_end(k)
+        
+        return data_dict
+
+
+def get_request_file_for_model(requests_path, model_name):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if (
+                req_content["status"] in ["FINISHED"]
+            ):
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_raw_eval_results(results_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError as e:  # not all eval values present
+            raise e
+            continue
+    return results

src/pages/about.py

@@ -0,0 +1,7 @@
+import gradio as gr
+
+from src.about import LLM_BENCHMARKS_TEXT
+
+def show_about_page(index: int):
+    with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=index):
+        gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

src/pages/result_table.py

@@ -0,0 +1,41 @@
+import gradio as gr
+from gradio_leaderboard import Leaderboard, SearchColumns, SelectColumns, ColumnFilter
+from src.leaderboard.read_evals import FILTERED_ONLY_FIELD
+from src.envs import EVAL_RESULTS_PATH
+from src.populate import get_leaderboard_df
+from src.display.utils import (
+    AutoEvalColumn,
+    fields,
+)
+
+def show_result_page(root_path: str, title: str, index: int):
+    raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH + f"/{root_path}")
+    leaderboard_df = original_df.copy()
+    with gr.TabItem(title, elem_id="llm-benchmark-tab-table", id=index):
+        return Leaderboard(
+            value=leaderboard_df,
+            datatype=[c.type for c in fields(AutoEvalColumn)],
+            select_columns=SelectColumns(
+                default_selection=[c for c in list(original_df.keys())],
+                cant_deselect=[c.name for c in fields(AutoEvalColumn)],
+                label="Select Columns to show:",
+            ),
+            hide_columns=FILTERED_ONLY_FIELD,
+            search_columns=SearchColumns(
+                primary_column=AutoEvalColumn.model.name,
+                secondary_columns=[],
+                placeholder="🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                label="Search",
+            ),
+            filter_columns=[
+                ColumnFilter(
+                    'params',
+                    type="slider",
+                    min=0.01,
+                    max=150,
+                    label="Select the number of parameters (B)",
+                ),
+            ],
+            bool_checkboxgroup_label="Hide models",
+            interactive=False,
+        )

src/pages/submit.py

@@ -0,0 +1,60 @@
+from src.display.utils import EVAL_COLS, EVAL_TYPES
+from src.envs import EVAL_REQUESTS_PATH
+from src.populate import get_evaluation_queue_df
+from src.submission.submit import add_new_eval
+import gradio as gr
+
+def show_submit_page(index: int):
+    (
+        finished_eval_queue_df,
+        running_eval_queue_df,
+        pending_eval_queue_df,
+    ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+    with gr.TabItem("🚀 Submit! ", elem_id="llm-benchmark-tab-table", id=index):
+        
+        with gr.Column():
+            with gr.Accordion(
+                f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                open=False,
+            ):
+                with gr.Row():
+                    finished_eval_table = gr.components.Dataframe(
+                        value=finished_eval_queue_df,
+                        headers=EVAL_COLS,
+                        datatype=EVAL_TYPES,
+                        row_count=5,
+                    )
+
+            with gr.Accordion(
+                f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                open=False,
+            ):
+                with gr.Row():
+                    pending_eval_table = gr.components.Dataframe(
+                        value=pending_eval_queue_df,
+                        headers=EVAL_COLS,
+                        datatype=EVAL_TYPES,
+                        row_count=5,
+                    )
+
+        with gr.Row():
+            gr.Markdown("# ✉️✨ Submit your model!", elem_classes="markdown-text")
+
+        with gr.Row():
+            with gr.Column():
+                model_name_textbox = gr.Textbox(label="Huggingface Model")
+                link_to_model_blog = gr.Textbox(label="Model release blog / Technical report")
+
+        submit_button = gr.Button("Submit Model")
+        submission_result = gr.Markdown()
+        submit_button.click(
+            add_new_eval,
+            [
+                model_name_textbox,
+                link_to_model_blog
+            ],
+            submission_result,
+        )
+
+        with gr.Row():
+            gr.Markdown('# ✉️✨ Submit your task <a href="https://github.com">here!</a>', elem_classes="markdown-text")

src/populate.py

@@ -0,0 +1,50 @@
+import json
+import os
+
+import pandas as pd
+
+from src.display.utils import EvalQueueColumn
+from src.leaderboard.read_evals import get_raw_eval_results
+
+
+def get_leaderboard_df(results_path: str) -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    raw_data = get_raw_eval_results(results_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.round(decimals=2)
+    return raw_data, df
+
+
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation queues requestes"""
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -0,0 +1,99 @@
+import json
+import os
+import re
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+
+import huggingface_hub
+from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig
+from transformers.models.auto.tokenization_auto import AutoTokenizer
+
+def check_model_card(repo_id: str) -> tuple[bool, str]:
+    """Checks if the model card and license exist and have been filled"""
+    try:
+        card = ModelCard.load(repo_id)
+    except huggingface_hub.utils.EntryNotFoundError:
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+
+    # Enforce license metadata
+    if card.data.license is None:
+        if not ("license_name" in card.data and "license_link" in card.data):
+            return False, (
+                "License not found. Please add a license to your model card using the `license` metadata or a"
+                " `license_name`/`license_link` pair."
+            )
+
+    # Enforce card content
+    if len(card.text) < 200:
+        return False, "Please add a description to your model card, it is too short."
+
+    return True, ""
+
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    try:
+        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        if test_tokenizer:
+            try:
+                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+            except ValueError as e:
+                return (
+                    False,
+                    f"uses a tokenizer which is not in a transformers release: {e}",
+                    None
+                )
+            except Exception as e:
+                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
+        return True, None, config
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+            None
+        )
+
+    except Exception as e:
+        return False, "was not found on hub!", None
+
+
+def get_model_size(model_info: ModelInfo):
+    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError):
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 1
+    model_size = size_factor * model_size
+    return model_size
+
+def get_model_arch(model_info: ModelInfo):
+    """Gets the model architecture from the configuration"""
+    return model_info.config.get("architectures", "Unknown")
+
+def already_submitted_models(requested_models_dir: str) -> set[str]:
+    """Gather a list of already submitted models to avoid duplicates"""
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"):
+                    continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -0,0 +1,94 @@
+import json
+import os
+from datetime import datetime, timezone
+
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.submission.check_validity import (
+    already_submitted_models,
+    check_model_card,
+    get_model_size,
+)
+
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
+
+def add_new_eval(
+    model: str,
+    link: str,
+):
+    global REQUESTED_MODELS
+    global USERS_TO_SUBMISSION_DATES
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+    revision = "main"
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        return styled_error("Could not get your model information. Please fill it up properly.")
+
+    model_size = get_model_size(model_info=model_info)
+
+    # Were the model card and license filled?
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        return styled_error("Please select a license for your model")
+
+    modelcard_OK, error_msg = check_model_card(model)
+    if not modelcard_OK:
+        return styled_error(error_msg)
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "revision": revision,
+        "link": link,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "params": model_size,
+        "license": license,
+        "private": False,
+    }
+
+    # Check for duplicate submission
+    if f"{model}_{revision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted.")
+
+    print("Creating eval file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False.json"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # Remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )