update backend

.gitignore

@@ -1,2 +1,3 @@
 __pycache__/
-.DS_Store
+.DS_Store
+rl-trained_agents/

src/backend.py

@@ -5,6 +5,7 @@ import os
 import random
 import shutil
 import sys
+import time
 import zipfile
 from pathlib import Path
 from typing import Optional
@@ -217,8 +218,7 @@ def download_from_hub(
             shutil.rmtree(log_path)
         else:
             raise ValueError(
-                f"The folder {log_path} already exists, use --force to overwrite it, "
-                "or choose '--exp-id 0' to create a new folder"
+                f"The folder {log_path} already exists, use --force to overwrite it, " "or choose '--exp-id 0' to create a new folder"
             )
 
     logger.info(f"Saving to {log_path}")
@@ -518,14 +518,9 @@ RESULTS_REPO = "open-rl-leaderboard/results_v2"
 
 def _backend_routine():
     # List only the text classification models
-    sb3_models = [
-        (model.modelId, model.sha) for model in API.list_models(filter=["reinforcement-learning", "stable-baselines3"])
-    ]
+    sb3_models = [(model.modelId, model.sha) for model in API.list_models(filter=["reinforcement-learning", "stable-baselines3"])]
     logger.info(f"Found {len(sb3_models)} SB3 models")
-    # Get the results
-    dataset = load_dataset(
-        RESULTS_REPO, split="train", download_mode="force_redownload", verification_mode="no_checks"
-    )
+    dataset = load_dataset(RESULTS_REPO, split="train", download_mode="force_redownload", verification_mode="no_checks")
     evaluated_models = [("/".join([x["user_id"], x["model_id"]]), x["sha"]) for x in dataset]
     pending_models = list(set(sb3_models) - set(evaluated_models))
     logger.info(f"Found {len(pending_models)} pending models")
@@ -533,8 +528,11 @@ def _backend_routine():
     if len(pending_models) == 0:
         return None
 
+    # Shuffle the dataset
+    random.shuffle(pending_models)
+
     # Select a random model
-    repo_id, sha = random.choice(pending_models)
+    repo_id, sha = pending_models.pop()
     user_id, model_id = repo_id.split("/")
     row = {"model_id": model_id, "user_id": user_id, "sha": sha}
 
@@ -555,17 +553,18 @@ def _backend_routine():
             row["episodic_returns"] = episodic_returns
         except Exception as e:
             logger.error(f"Error evaluating {model_id}: {e}")
+            logger.exception(e)
             row["status"] = "FAILED"
 
     else:
         logger.error(f"No environment found for {model_id}")
         row["status"] = "FAILED"
 
-    dataset = load_dataset(
-        RESULTS_REPO, split="train", download_mode="force_redownload", verification_mode="no_checks"
-    )  # Reload the dataset, in case it was updated
+    # load the last version of the dataset
+    dataset = load_dataset(RESULTS_REPO, split="train", download_mode="force_redownload", verification_mode="no_checks")
     dataset = dataset.add_item(row)
     dataset.push_to_hub(RESULTS_REPO, split="train")
+    time.sleep(60)  # Sleep for 1 minute to avoid rate limiting
 
 
 def backend_routine():
@@ -573,6 +572,7 @@ def backend_routine():
         _backend_routine()
     except Exception as e:
         logger.error(f"{e.__class__.__name__}: {str(e)}")
+        logger.exception(e)
 
 
 if __name__ == "__main__":