Added 4bit support

main_backend_lighteval.py

@@ -52,16 +52,20 @@ def run_auto_eval():
     eval_request = eval_requests[0]
     pp.pprint(eval_request)
 
+    params_size = eval_request.params
+    if eval_request.precision == '4bit':
+        params_size //= 2
+    
     # For GPU
-    if not eval_request or eval_request.params < 0:
+    if not eval_request or params_size < 0:
         raise ValueError("Couldn't detect number of params, please make sure the metadata is available")
-    elif eval_request.params < 4:
+    elif params_size < 4:
         instance_size, instance_type, cap = "x1", "nvidia-t4", 40
-    elif eval_request.params < 9:
+    elif params_size < 9:
         instance_size, instance_type, cap = "x1", "nvidia-a10g", 40
-    elif eval_request.params < 30:
+    elif params_size < 30:
         instance_size, instance_type, cap = "x4", "nvidia-a10g", 20
-    elif eval_request.params < 45:
+    elif params_size < 45:
         instance_size, instance_type, cap = "x2", "nvidia-a100", 5
     else: 
         set_eval_request(