connect front and backend

backend-cli.py

@@ -2,6 +2,7 @@
 
 import os
 import json
+import argparse
 
 import socket
 import random
@@ -33,7 +34,8 @@ def my_set_eval_request(api, eval_request, set_to_status, hf_repo, local_dir):
         try:
             set_eval_request(api=api, eval_request=eval_request, set_to_status=set_to_status, hf_repo=hf_repo, local_dir=local_dir)
             return
-        except Exception:
+        except Exception as e:
+            print(f"Error setting eval request to {set_to_status}: {e}. Retrying in 60 seconds")
             time.sleep(60)
     return
 
@@ -262,14 +264,21 @@ def process_pending_requests() -> bool:
     return True
 
 
+def get_args():
+    parser = argparse.ArgumentParser(description='Run the backend')
+    parser.add_argument('--debug', action='store_true', help='Run in debug mode')
+    return parser.parse_args()
+
+
 if __name__ == "__main__":
-    local_debug = True
+    args = get_args()
+    local_debug = args.debug
     #debug specific task by ping
     if local_debug:
         debug_model_names = ['mistralai/Mixtral-8x7B-Instruct-v0.1']
         # debug_model_names = ["TheBloke/Mixtral-8x7B-v0.1-GPTQ"]
         # debug_task_name = 'ifeval'
-        debug_task_name = 'selfcheckgpt'
+        debug_task_name = 'mmlu'
         task_lst = TASKS_HARNESS.copy()
         for task in task_lst:
             for debug_model_name in debug_model_names:
@@ -279,31 +288,24 @@ if __name__ == "__main__":
                 eval_request = EvalRequest(model=debug_model_name, private=False, status='', json_filepath='', precision='float16')
                 results = process_evaluation(task, eval_request)
 
-    wait = True
-    hard_task_lst = None
-    if socket.gethostname() in {'hamburg', 'neuromancer'} or os.path.isdir("/home/pminervi"):
-        wait = False
-        hard_task_lst = ['nq', 'trivia', 'tqa']
-
-    if wait:
-        time.sleep(60 * random.randint(5, 10))
+    while True:
+        res = False
 
-    res = False
-
-    if random.randint(0, 10) == 0:
+        # if random.randint(0, 10) == 0:
         res = process_pending_requests()
+        print(f"waiting for 60 seconds")
         time.sleep(60)
 
-    if res is False:
-        if random.randint(0, 5) == 0:
-            res = maybe_refresh_results(100, hard_task_lst=hard_task_lst)
-        else:
-            res = process_finished_requests(100, hard_task_lst=hard_task_lst)
+        # if res is False:
+        #     if random.randint(0, 5) == 0:
+        #         res = maybe_refresh_results(100)
+        #     else:
+        #         res = process_finished_requests(100)
 
-    time.sleep(60)
+        # time.sleep(60)
 
-    if res is False:
-        if random.randint(0, 5) == 0:
-            res = maybe_refresh_results(0, hard_task_lst=hard_task_lst)
-        else:
-            res = process_finished_requests(0, hard_task_lst=hard_task_lst)
+        # if res is False:
+        #     if random.randint(0, 5) == 0:
+        #         res = maybe_refresh_results(0)
+        #     else:
+        #         res = process_finished_requests(0)

src/backend/envs.py

@@ -35,7 +35,7 @@ class Tasks(Enum):
     # task8 = Task("xsum", "rougeL", "XSum", 2)
     # task9 = Task("cnndm", "rougeL", "CNN/DM", 2)
 
-    task8_1 = Task("xsum_v2", "rougeL", "XSum", 0)
+    # task8_1 = Task("xsum_v2", "rougeL", "XSum", 0)
     # task9_1 = Task("cnndm_v2", "rougeL", "CNN/DM", 0)
 
     # task10 = Task("memo-trap", "acc", "memo-trap", 0)
@@ -43,7 +43,7 @@ class Tasks(Enum):
 
     # task13 = Task("ifeval", "prompt_level_strict_acc", "IFEval", 0)
 
-    # task14 = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT", 0)
+    task14 = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT", 0)
 
     # task15 = Task("fever10", "acc", "FEVER", 16)
     # task15_1 = Task("fever11", "acc", "FEVER", 8)
@@ -56,7 +56,7 @@ class Tasks(Enum):
     # task19 = Task("faithdial_hallu_v2", "acc", "FaithDial", 8)
 
     # task20 = Task("race", "acc", "RACE", 0)
-    task21 = Task("gsm8k", "acc", "GSM8K", 0)
+    task21 = Task("mmlu", "acc", "MMLU", 5)
 
 
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")

src/backend/moe_infinity.py

@@ -1,7 +1,6 @@
 import torch
 import os
 from transformers import AutoTokenizer
-import transformers
 from transformers import AutoModelForCausalLM
 from moe_infinity import MoE
 from typing import List, Tuple, Optional, Union
@@ -29,7 +28,7 @@ class MoEHFLM(HFLM):
         self.use_chat_template = use_chat_template
         if "device" in kwargs:
             kwargs.pop("device")
-        super().__init__(*args, **kwargs, pretrained=pretrained, device="cuda:0")  # Assuming HFLM accepts a 'pretrained' arg and handles it
+        super().__init__(*args, **kwargs, pretrained=pretrained, device_map="cuda:0")  # Assuming HFLM accepts a 'pretrained' arg and handles it
         # self._create_model()
 
     def _create_model(self, *args, **kwargs):
@@ -43,7 +42,8 @@ class MoEHFLM(HFLM):
         }
         # Update default config with any user-provided config
         final_moe_config = {**default_moe_config, **self.moe_config}
-        self._model = MoE(self.checkpoint, final_moe_config)
+        # self._model = MoE(self.checkpoint, final_moe_config)
+        self._model = AutoModelForCausalLM.from_pretrained(self.checkpoint, torch_dtype=torch.float16, device_map="auto")
 
     @property
     def max_length(self):

src/backend/run_eval_suite.py

@@ -33,6 +33,7 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
 
     print(f"Selected Tasks: {task_names}")
     print(f"Eval Request: {eval_request.get_model_args()}")
+    print(f"Num Fewshot: {num_fewshot}, Batch Size: {batch_size}, Device: {device}, Use Cache: {use_cache}, Limit: {limit}")
     # hf-chat is implemented to use apply_chat_template
     results = evaluator.simple_evaluate(model="moe-infinity",  # "hf-causal-experimental",  # "hf-causal", hf-chat
                                         model_args=eval_request.get_model_args(),

src/display/utils.py

@@ -24,7 +24,7 @@ class Tasks(Enum):
     # truthfulqa_mc2 = Task("truthfulqa_mc2", "acc", "TruthQA MC2/Acc")
     # truthfulqa_gen = Task("truthfulqa_gen", "rougeL_acc", "TruthQA Gen/ROUGE")
 
-    xsum_r = Task("xsum_v2", "rougeL", "XSum/ROUGE")
+    # xsum_r = Task("xsum_v2", "rougeL", "XSum/ROUGE")
     # xsum_f = Task("xsum_v2", "factKB", "XSum/factKB")
     # xsum_b = Task("xsum_v2", "bertscore_precision", "XSum/BERT-P")
 
@@ -45,8 +45,8 @@ class Tasks(Enum):
     # halueval_dial = Task("halueval_dialogue", "acc", "HaluDial/Acc")
 
     # # XXX include me back at some point
-    # selfcheck = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT")
-    gsm8k = Task("gsm8k", "acc", "GSM8K")
+    selfcheck = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT")
+    mmlu = Task("hendrycksTest", "acc", "MMLU")
 
 
 # These classes are for user facing column names,
@@ -62,7 +62,8 @@ class ColumnContent:
     dummy: bool = False
 
 auto_eval_column_dict = []
-auto_eval_column_dict.append(["system", ColumnContent, ColumnContent("System", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["hardware", ColumnContent, ColumnContent("Hardware", "str", True, never_hidden=True)])
 # Init
 # auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 # auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])