Create lock for global variables

app.py

@@ -102,6 +102,11 @@ def submit_results():
     leaderboard_server.save_pre_submit()
     leaderboard_server.update_leaderboard()
     gr.Info('Submission successful!')
+    
+    with leaderboard_server.var_lock.ro:
+        leaderboard = leaderboard_server.get_leaderboard(category=leaderboard_server.TASKS_CATEGORY_OVERALL)
+        submission_ids = leaderboard_server.submission_ids
+    
     return (
         gr.update(value='Pre-submit model', visible=True, interactive=True),
         gr.update(visible=False),
@@ -109,10 +114,10 @@ def submit_results():
         gr.update(visible=False),
         gr.update(visible=False),
         gr.update(visible=False),
-        gr.DataFrame(value=leaderboard_server.get_leaderboard(category=leaderboard_server.tasks_category_overall), visible=True),
+        gr.DataFrame(value=leaderboard, visible=True),
         gr.update(visible=False),
-        gr.update(choices=leaderboard_server.submission_ids),
-        gr.update(value=leaderboard_server.tasks_category_overall),
+        gr.update(choices=submission_ids),
+        gr.update(value=leaderboard_server.TASKS_CATEGORY_OVERALL),
     )
 
 
@@ -155,11 +160,12 @@ def hide_modal():
 
 def on_application_load():
     leaderboard_server.update_leaderboard()
-    return (
-        gr.DataFrame(value=leaderboard_server.get_leaderboard(category=leaderboard_server.tasks_category_overall), visible=True),
-        gr.update(choices=leaderboard_server.submission_ids),
-        gr.update(value=leaderboard_server.tasks_category_overall),
-    )
+    with leaderboard_server.var_lock.ro:
+        return (
+            gr.DataFrame(value=leaderboard_server.get_leaderboard(category=leaderboard_server.TASKS_CATEGORY_OVERALL), visible=True),
+            gr.update(choices=leaderboard_server.submission_ids),
+            gr.update(value=leaderboard_server.TASKS_CATEGORY_OVERALL),
+        )
 
 
 custom_css = """
@@ -228,15 +234,15 @@ with gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main:
 
                 with gr.Row():
                     category_of_tasks = gr.Dropdown(
-                        choices=[leaderboard_server.tasks_category_overall] + list(leaderboard_server.tasks_categories),
-                        value=leaderboard_server.tasks_category_overall,
+                        choices=[leaderboard_server.TASKS_CATEGORY_OVERALL] + list(leaderboard_server.TASKS_CATEGORIES),
+                        value=leaderboard_server.TASKS_CATEGORY_OVERALL,
                         label="Category of benchmarks",
                         interactive=True,
                     )
                 
                 with gr.Row():
                     results_table = gr.DataFrame(
-                        leaderboard_server.get_leaderboard(category=leaderboard_server.tasks_category_overall),
+                        leaderboard_server.get_leaderboard(category=leaderboard_server.TASKS_CATEGORY_OVERALL),
                         interactive=False,
                         label=None,
                         visible=True,
@@ -252,11 +258,13 @@ with gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main:
 
         with gr.TabItem('Model details'):
             gr.Markdown(MORE_DETAILS_MARKDOWN)
-            detail_dropdown = gr.Dropdown(
-                choices=leaderboard_server.submission_ids,  # TODO: team_name/model_name
-                label="Select model",
-                interactive=True,
-            )
+            
+            with leaderboard_server.var_lock.ro:
+                detail_dropdown = gr.Dropdown(
+                    choices=leaderboard_server.submission_ids,  # TODO: team_name/model_name
+                    label="Select model",
+                    interactive=True,
+                )
 
             with gr.Row():
                 model_description = gr.Text(value='', label='Model description', visible=False, interactive=False)
@@ -327,7 +335,7 @@ with gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main:
                 pre_submission_btn.click(
                     fn=on_submit_pressed,
                     outputs=[pre_submission_btn],
-                ).then(
+                ).then(  # TODO: Zjistit proč to neběží konkurentně.
                     fn=process_submission,
                     inputs=list(submission_inputs.values()),
                     outputs=[

server.py

@@ -102,6 +102,46 @@ def check_significance(model_a_path, model_b_path):
 
 pre_submit_lock = Lock()
 
+class _ReadLock:
+    def __init__(self, lock):
+       self._lock = lock
+       self.reading = 0
+    
+    def __enter__(self):
+        with self._lock:
+            self.reading += 1
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        with self._lock:
+            self.reading -= 1
+
+class ReadWriteLock:
+    """
+    Zámek, který ověří, že nikdo nečte když se zapisuje a že zapisuje pouze jeden
+    """
+    
+    def __init__(self):
+       self._lock = Lock()
+       self.ro = _ReadLock(self._lock)
+       self.rw = self
+    
+    def __enter__(self):
+        self._lock.acquire()
+        while True:
+            reading = self.ro.reading
+            if reading > 0:
+                self._lock.release()
+                time.sleep(1)
+                self._lock.acquire()
+            elif reading < 0:
+                self._lock.release()
+                raise RuntimeError()
+            else:
+                return
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._lock.release()
+
 class LeaderboardServer:
     def __init__(self):
         self.server_address = REPO
@@ -112,13 +152,16 @@ class LeaderboardServer:
             token=HF_TOKEN,
             local_dir="./",
         )
-        self.submission_id_to_file = {}  # Map submission ids to file paths
-        self.tasks_metadata = json.load(open(TASKS_METADATA_PATH))
-        self.tasks_categories = {self.tasks_metadata[task]["category"] for task in self.tasks_metadata}
-        self.tasks_category_overall = "Overall"
+        self.TASKS_METADATA = json.load(open(TASKS_METADATA_PATH))
+        self.TASKS_CATEGORIES = {self.TASKS_METADATA[task]["category"] for task in self.TASKS_METADATA}
+        self.TASKS_CATEGORY_OVERALL = "Overall"
+        
+        self.var_lock = ReadWriteLock()
         self.submission_ids = set()
+        self.submission_id_to_file = {}  # Map submission ids to file paths
         self.fetch_existing_models()
         self.tournament_results = self.load_tournament_results()
+        
         self.pre_submit_lock = pre_submit_lock
         self.pre_submit = None
 
@@ -130,7 +173,9 @@ class LeaderboardServer:
             local_dir="./",
         )
         self.fetch_existing_models()
-        self.tournament_results = self.load_tournament_results()
+        
+        with self.var_lock.rw:
+            self.tournament_results = self.load_tournament_results()
 
     def load_tournament_results(self):
         metadata_rank_paths = os.path.join(self.local_leaderboard, "tournament.json")
@@ -148,171 +193,174 @@ class LeaderboardServer:
             if metadata is None:
                 continue
             submission_id = metadata["submission_id"]
-            self.submission_ids.add(submission_id)
-
-            self.submission_id_to_file[submission_id] = submission_file
+            
+            with self.var_lock.rw:
+                self.submission_ids.add(submission_id)
+                self.submission_id_to_file[submission_id] = submission_file
 
     def get_leaderboard(self, pre_submit=None, category=None):
-        tournament_results = pre_submit.tournament_results if pre_submit else self.tournament_results
-        category = category if category else self.tasks_category_overall
+        with self.var_lock.ro:
+            tournament_results = pre_submit.tournament_results if pre_submit else self.tournament_results
+            category = category if category else self.TASKS_CATEGORY_OVERALL
 
-        if len(tournament_results) == 0:
-            return pd.DataFrame(columns=['No submissions yet'])
-        else:
-            processed_results = []
-            for submission_id in tournament_results.keys():
-                path = self.submission_id_to_file.get(submission_id)
-                if path is None:
-                    if pre_submit and submission_id == pre_submit.submission_id:
-                        data = json.load(open(pre_submit.file))
+            if len(tournament_results) == 0:
+                return pd.DataFrame(columns=['No submissions yet'])
+            else:
+                processed_results = []
+                for submission_id in tournament_results.keys():
+                    path = self.submission_id_to_file.get(submission_id)
+                    if path is None:
+                        if pre_submit and submission_id == pre_submit.submission_id:
+                            data = json.load(open(pre_submit.file))
+                        else:
+                            raise gr.Error(f"Internal error: Submission [{submission_id}] not found")
+                    elif path:
+                        data = json.load(open(path))
                     else:
-                        raise gr.Error(f"Internal error: Submission [{submission_id}] not found")
-                elif path:
-                    data = json.load(open(path))
-                else:
-                    raise gr.Error(f"Submission [{submission_id}] not found")
-                
-                if submission_id != data["metadata"]["submission_id"]:
-                    raise gr.Error(f"Proper submission [{submission_id}] not found")
-
-                local_results = {}
-                win_score = {}
-                visible_metrics_map_word_to_header = {}
-                for task in self.tasks_metadata.keys():
+                        raise gr.Error(f"Submission [{submission_id}] not found")
                     
-                    task_category = self.tasks_metadata[task]["category"]
-                    if category not in (self.tasks_category_overall, task_category):
-                        continue
-                    else:
-                        # tournament_results
-                        num_of_competitors = 0
-                        num_of_wins = 0
-                        for competitor_id in tournament_results[submission_id].keys() - {submission_id}: # without self
-                            num_of_competitors += 1
-                            if tournament_results[submission_id][competitor_id][task]:
-                                num_of_wins += 1
-                        task_score = num_of_wins / num_of_competitors * 100 if num_of_competitors > 0 else 100
-                        win_score.setdefault(task_category, []).append(task_score)
+                    if submission_id != data["metadata"]["submission_id"]:
+                        raise gr.Error(f"Proper submission [{submission_id}] not found")
+
+                    local_results = {}
+                    win_score = {}
+                    visible_metrics_map_word_to_header = {}
+                    for task in self.TASKS_METADATA.keys():
                         
-                        if category == task_category:
-                            local_results[task] = task_score
-                            for metric in VISIBLE_METRICS:
-                                visible_metrics_map_word_to_header[task + "_" + metric] = self.tasks_metadata[task]["abbreviation"] + " " + metric
-                                metric_value = data['results'][task].get(metric)
-                                if metric_value is not None:
-                                    local_results[task + "_" + metric] = metric_value * 100
-                                    break  # Only the first metric of every task
-                
-                
-                for c in win_score:
-                    win_score[c] = sum(win_score[c]) / len(win_score[c])
-                
-                if category == self.tasks_category_overall:
+                        task_category = self.TASKS_METADATA[task]["category"]
+                        if category not in (self.TASKS_CATEGORY_OVERALL, task_category):
+                            continue
+                        else:
+                            # tournament_results
+                            num_of_competitors = 0
+                            num_of_wins = 0
+                            for competitor_id in tournament_results[submission_id].keys() - {submission_id}: # without self
+                                num_of_competitors += 1
+                                if tournament_results[submission_id][competitor_id][task]:
+                                    num_of_wins += 1
+                            task_score = num_of_wins / num_of_competitors * 100 if num_of_competitors > 0 else 100
+                            win_score.setdefault(task_category, []).append(task_score)
+                            
+                            if category == task_category:
+                                local_results[task] = task_score
+                                for metric in VISIBLE_METRICS:
+                                    visible_metrics_map_word_to_header[task + "_" + metric] = self.TASKS_METADATA[task]["abbreviation"] + " " + metric
+                                    metric_value = data['results'][task].get(metric)
+                                    if metric_value is not None:
+                                        local_results[task + "_" + metric] = metric_value * 100
+                                        break  # Only the first metric of every task
+                    
+                    
                     for c in win_score:
-                        local_results[c] = win_score[c]
-                    local_results["average_score"] = sum(win_score.values()) / len(win_score)
-                else:
-                    local_results["average_score"] = win_score[category]
-                
-                model_link = data["metadata"]["link_to_model"]
-                model_title = data["metadata"]["team_name"] + "/" + data["metadata"]["model_name"]
-                model_title_abbr = self.abbreviate(data["metadata"]["team_name"], 14) + "/" + self.abbreviate(data["metadata"]["model_name"], 14)
-                local_results["model"] = f'<a href={xmlQuoteAttr(model_link)} title={xmlQuoteAttr(model_title)}>{xmlEscape(model_title_abbr, MARKDOWN_SPECIAL_CHARACTERS)}</a>'
-                release = data["metadata"].get("submission_timestamp")
-                release = time.strftime("%Y-%m-%d", time.gmtime(release)) if release else "N/A"
-                local_results["release"] = release
-                local_results["model_type"] = data["metadata"]["model_type"]
-                local_results["parameters"] = data["metadata"]["parameters"]
+                        win_score[c] = sum(win_score[c]) / len(win_score[c])
+                    
+                    if category == self.TASKS_CATEGORY_OVERALL:
+                        for c in win_score:
+                            local_results[c] = win_score[c]
+                        local_results["average_score"] = sum(win_score.values()) / len(win_score)
+                    else:
+                        local_results["average_score"] = win_score[category]
+                    
+                    model_link = data["metadata"]["link_to_model"]
+                    model_title = data["metadata"]["team_name"] + "/" + data["metadata"]["model_name"]
+                    model_title_abbr = self.abbreviate(data["metadata"]["team_name"], 14) + "/" + self.abbreviate(data["metadata"]["model_name"], 14)
+                    local_results["model"] = f'<a href={xmlQuoteAttr(model_link)} title={xmlQuoteAttr(model_title)}>{xmlEscape(model_title_abbr, MARKDOWN_SPECIAL_CHARACTERS)}</a>'
+                    release = data["metadata"].get("submission_timestamp")
+                    release = time.strftime("%Y-%m-%d", time.gmtime(release)) if release else "N/A"
+                    local_results["release"] = release
+                    local_results["model_type"] = data["metadata"]["model_type"]
+                    local_results["parameters"] = data["metadata"]["parameters"]
+                    
+                    if pre_submit and submission_id == pre_submit.submission_id:
+                        processed_results.insert(0, local_results)
+                    else:
+                        processed_results.append(local_results)
+                dataframe = pd.DataFrame.from_records(processed_results)
                 
-                if pre_submit and submission_id == pre_submit.submission_id:
-                    processed_results.insert(0, local_results)
-                else:
-                    processed_results.append(local_results)
-            dataframe = pd.DataFrame.from_records(processed_results)
-            
-            extra_attributes_map_word_to_header = {
-                "model": "Model",
-                "release": "Release",
-                "average_score": "Average ⬆️",
-                "team_name": "Team name",
-                "model_name": "Model name",
-                "model_type": "Type",
-                "parameters": "# θ (B)",
-                "input_length": "Input length (# tokens)",
-                "precision": "Precision",
-                "description": "Description",
-                "link_to_model": "Link to model"
-            }
-            first_attributes = [
-                "model",
-                "release",
-                "model_type",
-                "parameters",
-                "average_score",
-            ]
-            df_order = [
-                key
-                for key in dict.fromkeys(
-                    first_attributes
-                    + list(self.tasks_metadata.keys())
-                    + list(dataframe.columns)
-                ).keys()
-                if key in dataframe.columns
-            ]
-            dataframe = dataframe[df_order]
-            attributes_map_word_to_header = {key: value["abbreviation"] for key, value in self.tasks_metadata.items()}
-            attributes_map_word_to_header.update(extra_attributes_map_word_to_header)
-            attributes_map_word_to_header.update(visible_metrics_map_word_to_header)
-            dataframe = dataframe.rename(
-                columns=attributes_map_word_to_header
-            )
-            return dataframe
+                extra_attributes_map_word_to_header = {
+                    "model": "Model",
+                    "release": "Release",
+                    "average_score": "Average ⬆️",
+                    "team_name": "Team name",
+                    "model_name": "Model name",
+                    "model_type": "Type",
+                    "parameters": "# θ (B)",
+                    "input_length": "Input length (# tokens)",
+                    "precision": "Precision",
+                    "description": "Description",
+                    "link_to_model": "Link to model"
+                }
+                first_attributes = [
+                    "model",
+                    "release",
+                    "model_type",
+                    "parameters",
+                    "average_score",
+                ]
+                df_order = [
+                    key
+                    for key in dict.fromkeys(
+                        first_attributes
+                        + list(self.TASKS_METADATA.keys())
+                        + list(dataframe.columns)
+                    ).keys()
+                    if key in dataframe.columns
+                ]
+                dataframe = dataframe[df_order]
+                attributes_map_word_to_header = {key: value["abbreviation"] for key, value in self.TASKS_METADATA.items()}
+                attributes_map_word_to_header.update(extra_attributes_map_word_to_header)
+                attributes_map_word_to_header.update(visible_metrics_map_word_to_header)
+                dataframe = dataframe.rename(
+                    columns=attributes_map_word_to_header
+                )
+                return dataframe
 
     def start_tournament(self, new_submission_id, new_model_file):
-        new_tournament = copy.deepcopy(self.tournament_results)
-        new_tournament[new_submission_id] = {}
-        new_tournament[new_submission_id][new_submission_id] = {
-            task: False for task in self.tasks_metadata.keys()
-        }
-        
-        rest_of_competitors = list(self.submission_ids - {new_submission_id})  # without self
-        num_of_competitors = len(rest_of_competitors)
-        
-        result_url = {}
-        result_inverse_url = {}
-        
-        while rest_of_competitors:
-            next_competitors = []
-            while rest_of_competitors:
-                if len(next_competitors) < 5:  # 5*2==10 tasks
-                    next_competitors.append(rest_of_competitors.pop())
-                else:
-                    break
+        with self.var_lock.ro:
+            new_tournament = copy.deepcopy(self.tournament_results)
+            new_tournament[new_submission_id] = {}
+            new_tournament[new_submission_id][new_submission_id] = {
+                task: False for task in self.TASKS_METADATA.keys()
+            }
             
-            for competitor_id in next_competitors:
-                result_url[competitor_id] = check_significance_send_task(new_model_file, self.submission_id_to_file[competitor_id])
-                result_inverse_url[competitor_id] = check_significance_send_task(self.submission_id_to_file[competitor_id], new_model_file)
+            rest_of_competitors = list(self.submission_ids - {new_submission_id})  # without self
+            num_of_competitors = len(rest_of_competitors)
             
-            while next_competitors:
-                competitor_id = next_competitors.pop(0)
-                result = check_significance_wait_for_result(result_url.pop(competitor_id))
-                result_inverse = check_significance_wait_for_result(result_inverse_url.pop(competitor_id))
-                
-                if rest_of_competitors:
-                    new_competitor_id = rest_of_competitors.pop()
-                    next_competitors.append(new_competitor_id)
-                    result_url[new_competitor_id] = check_significance_send_task(new_model_file, self.submission_id_to_file[new_competitor_id])
-                    result_inverse_url[new_competitor_id] = check_significance_send_task(self.submission_id_to_file[new_competitor_id], new_model_file)
+            result_url = {}
+            result_inverse_url = {}
+            
+            while rest_of_competitors:
+                next_competitors = []
+                while rest_of_competitors:
+                    if len(next_competitors) < 5:  # 5*2==10 tasks
+                        next_competitors.append(rest_of_competitors.pop())
+                    else:
+                        break
                 
-                new_tournament[new_submission_id][competitor_id] = {
-                    task: data["significant"] for task, data in result.items()
-                }
-                new_tournament[competitor_id][new_submission_id] = {
-                    task: data["significant"] for task, data in result_inverse.items()
-                }
+                for competitor_id in next_competitors:
+                    result_url[competitor_id] = check_significance_send_task(new_model_file, self.submission_id_to_file[competitor_id])
+                    result_inverse_url[competitor_id] = check_significance_send_task(self.submission_id_to_file[competitor_id], new_model_file)
                 
-                num_of_competitors_done = num_of_competitors - len(next_competitors) - len(rest_of_competitors)
-                gr.Info(f"Tournament: {num_of_competitors_done}/{num_of_competitors} = {(num_of_competitors_done) * 100 // num_of_competitors}% done")
+                while next_competitors:
+                    competitor_id = next_competitors.pop(0)
+                    result = check_significance_wait_for_result(result_url.pop(competitor_id))
+                    result_inverse = check_significance_wait_for_result(result_inverse_url.pop(competitor_id))
+                    
+                    if rest_of_competitors:
+                        new_competitor_id = rest_of_competitors.pop()
+                        next_competitors.append(new_competitor_id)
+                        result_url[new_competitor_id] = check_significance_send_task(new_model_file, self.submission_id_to_file[new_competitor_id])
+                        result_inverse_url[new_competitor_id] = check_significance_send_task(self.submission_id_to_file[new_competitor_id], new_model_file)
+                    
+                    new_tournament[new_submission_id][competitor_id] = {
+                        task: data["significant"] for task, data in result.items()
+                    }
+                    new_tournament[competitor_id][new_submission_id] = {
+                        task: data["significant"] for task, data in result_inverse.items()
+                    }
+                    
+                    num_of_competitors_done = num_of_competitors - len(next_competitors) - len(rest_of_competitors)
+                    gr.Info(f"Tournament: {num_of_competitors_done}/{num_of_competitors} = {(num_of_competitors_done) * 100 // num_of_competitors}% done")
         
         return new_tournament
 
@@ -416,7 +464,8 @@ class LeaderboardServer:
                 self.pre_submit = None
 
     def get_model_detail(self, submission_id):
-        path = self.submission_id_to_file.get(submission_id)
+        with self.var_lock.ro:
+            path = self.submission_id_to_file.get(submission_id)
         if path is None:
             raise gr.Error(f"Submission [{submission_id}] not found")
         data = json.load(open(path))