in deployment mode, check whethet the cache_dir exists

app.py

@@ -14,7 +14,7 @@
 
 import logging
 from os import mkdir
-from os.path import isdir
+from os.path import exists, isdir
 from pathlib import Path
 
 import streamlit as st
@@ -143,63 +143,63 @@ def load_or_prepare_widgets(ds_args, show_embeddings, use_cache=False):
 
     """
 
-    if not isdir(CACHE_DIR):
-        logs.warning("Creating cache")
-        # We need to preprocess everything.
-        # This should eventually all go into a prepare_dataset CLI
-        mkdir(CACHE_DIR)
     if use_cache:
         logs.warning("Using cache")
-    try:
+    if True:
+    #try:
         dstats = dataset_statistics.DatasetStatisticsCacheClass(CACHE_DIR, **ds_args, use_cache=use_cache)
         # Don't recalculate; we're live
         dstats.set_deployment(True)
-    except:
-        logs.warning("We're screwed")
-    try:
-        # We need to have the text_dset loaded for further load_or_prepare
-        dstats.load_or_prepare_dataset()
-    except:
-        logs.warning("Missing a cache for load or prepare dataset")
-    try:
-        # Header widget
-        dstats.load_or_prepare_dset_peek()
-    except:
-        logs.warning("Missing a cache for dset peek")
-    try:
-        # General stats widget
-        dstats.load_or_prepare_general_stats()
-    except:
-        logs.warning("Missing a cache for general stats")
-    try:
-        # Labels widget
-        dstats.load_or_prepare_labels()
-    except:
-        logs.warning("Missing a cache for prepare labels")
-    try:
-        # Text lengths widget
-        dstats.load_or_prepare_text_lengths()
-    except:
-        logs.warning("Missing a cache for text lengths")
-    if show_embeddings:
+        # checks whether the cache_dir exists in deployment mode
+        # creates cache_dir if not and if in development mode
+        cache_dir_exists = dstats.check_cache_dir()
+    #except:
+    #    logs.warning("We're screwed")
+    if cache_dir_exists:
         try:
-            # Embeddings widget
-            dstats.load_or_prepare_embeddings()
+            # We need to have the text_dset loaded for further load_or_prepare
+            dstats.load_or_prepare_dataset()
         except:
-            logs.warning("Missing a cache for embeddings")
-    try:
-        dstats.load_or_prepare_text_duplicates()
-    except:
-        logs.warning("Missing a cache for text duplicates")
-    try:
-        dstats.load_or_prepare_npmi()
-    except:
-        logs.warning("Missing a cache for npmi")
-    try:
-        dstats.load_or_prepare_zipf()
-    except:
-        logs.warning("Missing a cache for zipf")
-    return dstats
+            logs.warning("Missing a cache for load or prepare dataset")
+        try:
+            # Header widget
+            dstats.load_or_prepare_dset_peek()
+        except:
+            logs.warning("Missing a cache for dset peek")
+        try:
+            # General stats widget
+            dstats.load_or_prepare_general_stats()
+        except:
+            logs.warning("Missing a cache for general stats")
+        try:
+            # Labels widget
+            dstats.load_or_prepare_labels()
+        except:
+            logs.warning("Missing a cache for prepare labels")
+        try:
+            # Text lengths widget
+            dstats.load_or_prepare_text_lengths()
+        except:
+            logs.warning("Missing a cache for text lengths")
+        if show_embeddings:
+            try:
+                # Embeddings widget
+                dstats.load_or_prepare_embeddings()
+            except:
+                logs.warning("Missing a cache for embeddings")
+        try:
+            dstats.load_or_prepare_text_duplicates()
+        except:
+            logs.warning("Missing a cache for text duplicates")
+        try:
+            dstats.load_or_prepare_npmi()
+        except:
+            logs.warning("Missing a cache for npmi")
+        try:
+            dstats.load_or_prepare_zipf()
+        except:
+            logs.warning("Missing a cache for zipf")
+    return dstats, cache_dir_exists
 
 def show_column(dstats, ds_name_to_dict, show_embeddings, column_id):
     """
@@ -257,21 +257,33 @@ def main():
         dataset_args_left = st_utils.sidebar_selection(ds_name_to_dict, " A")
         dataset_args_right = st_utils.sidebar_selection(ds_name_to_dict, " B")
         left_col, _, right_col = st.columns([10, 1, 10])
-        dstats_left = load_or_prepare_widgets(
+        dstats_left, cache_exists_left = load_or_prepare_widgets(
             dataset_args_left, show_embeddings, use_cache=use_cache
         )
         with left_col:
-            show_column(dstats_left, ds_name_to_dict, show_embeddings, " A")
-        dstats_right = load_or_prepare_widgets(
+            if cache_exists_left:
+                show_column(dstats_left, ds_name_to_dict, show_embeddings, " A")
+            else:
+                st.markdown("### Missing pre-computed data measures!")
+                st.write(dataset_args_left)
+        dstats_right, cache_exists_right = load_or_prepare_widgets(
             dataset_args_right, show_embeddings, use_cache=use_cache
         )
         with right_col:
-            show_column(dstats_right, ds_name_to_dict, show_embeddings, " B")
+            if cache_exists_right:
+                show_column(dstats_right, ds_name_to_dict, show_embeddings, " B")
+            else:
+                st.markdown("### Missing pre-computed data measures!")
+                st.write(dataset_args_right)
     else:
         logs.warning("Using Single Dataset Mode")
         dataset_args = st_utils.sidebar_selection(ds_name_to_dict, "")
-        dstats = load_or_prepare_widgets(dataset_args, show_embeddings, use_cache=use_cache)
-        show_column(dstats, ds_name_to_dict, show_embeddings, "")
+        dstats, cache_exists = load_or_prepare_widgets(dataset_args, show_embeddings, use_cache=use_cache)
+        if cache_exists:
+            show_column(dstats, ds_name_to_dict, show_embeddings, "")
+        else:
+            st.markdown("### Missing pre-computed data measures!")
+            st.write(dataset_args)
 
 
 if __name__ == "__main__":

data_measurements/dataset_statistics.py

@@ -245,9 +245,6 @@ class DatasetStatisticsCacheClass:
             self.cache_dir,
             f"{dset_name}_{dset_config}_{split_name}_{text_field}",  # {label_field},
         )
-        if not isdir(self.cache_path):
-            logs.warning("Creating cache directory %s." % self.cache_path)
-            mkdir(self.cache_path)
 
         # Cache files not needed for UI
         self.dset_fid = pjoin(self.cache_path, "base_dset")
@@ -302,6 +299,21 @@ class DatasetStatisticsCacheClass:
         """
         self.live = live
 
+    def check_cache_dir(self):
+        """
+        First function to call to create the cache directory.
+        If in deployment mode and cache directory does not already exist,
+        return False.
+        """
+        if self.live:
+            return isdir(self.cache_path)
+        else:
+            if not isdir(self.cache_path):
+                logs.warning("Creating cache directory %s." % self.cache_path)
+                mkdir(self.cache_path)
+            return isdir(self.cache_path)
+
+
     def get_base_dataset(self):
         """Gets a pointer to the truncated base dataset object."""
         if not self.dset: