Merge branch 'main' of https://huggingface.co/spaces/huggingface/data-measurements-tool-2 into main

app.py

@@ -117,7 +117,10 @@ def load_or_prepare(ds_args, show_embeddings, use_cache=False):
         logs.warning("Loading Embeddings")
         dstats.load_or_prepare_embeddings()
     logs.warning("Loading nPMI")
-    dstats.load_or_prepare_npmi()
+    try:
+        dstats.load_or_prepare_npmi()
+    except:
+        logs.warning("Missing a cache for npmi")
     logs.warning("Loading Zipf")
     dstats.load_or_prepare_zipf()
     return dstats
@@ -147,25 +150,55 @@ def load_or_prepare_widgets(ds_args, show_embeddings, use_cache=False):
         mkdir(CACHE_DIR)
     if use_cache:
         logs.warning("Using cache")
-    dstats = dataset_statistics.DatasetStatisticsCacheClass(CACHE_DIR, **ds_args, use_cache=use_cache)
-    # Don't recalculate; we're live
-    dstats.set_deployment(True)
-    # We need to have the text_dset loaded for further load_or_prepare
-    dstats.load_or_prepare_dataset()
-    # Header widget
-    dstats.load_or_prepare_dset_peek()
-    # General stats widget
-    dstats.load_or_prepare_general_stats()
-    # Labels widget
-    dstats.load_or_prepare_labels()
-    # Text lengths widget
-    dstats.load_or_prepare_text_lengths()
+    try:
+        dstats = dataset_statistics.DatasetStatisticsCacheClass(CACHE_DIR, **ds_args, use_cache=use_cache)
+        # Don't recalculate; we're live
+        dstats.set_deployment(True)
+    except:
+        logs.warning("We're screwed")
+    try:
+        # We need to have the text_dset loaded for further load_or_prepare
+        dstats.load_or_prepare_dataset()
+    except:
+        logs.warning("Missing a cache for load or prepare dataset")
+    try:
+        # Header widget
+        dstats.load_or_prepare_dset_peek()
+    except:
+        logs.warning("Missing a cache for dset peek")
+    try:
+        # General stats widget
+        dstats.load_or_prepare_general_stats()
+    except:
+        logs.warning("Missing a cache for general stats")
+    try:
+        # Labels widget
+        dstats.load_or_prepare_labels()
+    except:
+        logs.warning("Missing a cache for prepare labels")
+    try:
+        # Text lengths widget
+        dstats.load_or_prepare_text_lengths()
+    except:
+        logs.warning("Missing a cache for text lengths")
     if show_embeddings:
-        # Embeddings widget
-        dstats.load_or_prepare_embeddings()
-    dstats.load_or_prepare_text_duplicates()
-    dstats.load_or_prepare_npmi()
-    dstats.load_or_prepare_zipf()
+        try:
+            # Embeddings widget
+            dstats.load_or_prepare_embeddings()
+        except:
+            logs.warning("Missing a cache for embeddings")
+    try:
+        dstats.load_or_prepare_text_duplicates()
+    except:
+        logs.warning("Missing a cache for text duplicates")
+    try:
+        dstats.load_or_prepare_npmi()
+    except:
+        logs.warning("Missing a cache for npmi")
+    try:
+        dstats.load_or_prepare_zipf()
+    except:
+        logs.warning("Missing a cache for zipf")
     return dstats
 
 def show_column(dstats, ds_name_to_dict, show_embeddings, column_id):

cache_dir/c4_realnewslike_train_text/text_dset/dataset.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9813f70c9be641905ca737aa8f16e29d6aa17155a76cd830e7a627aed91431f4
+size 529606944

cache_dir/c4_realnewslike_train_text/text_dset/dataset_info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff9f59542efc98b40f23b64408e3fbaed544ad8f0d1fb1e7126ead5af52844ac
+size 945

cache_dir/c4_realnewslike_train_text/text_dset/state.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2f6884f5ee381e5df2d267dae699aaf4792ba06c8f16830c9c19c144b4b3003
+size 256

data_measurements/dataset_statistics.py

@@ -498,7 +498,7 @@ class DatasetStatisticsCacheClass:
         if not self.live:
             if self.tokenized_df is None:
                 logs.warning("Tokenized dataset not yet loaded; doing so.")
-                self.load_or_prepare_dataset()
+                self.load_or_prepare_tokenized_df()
             if self.vocab_counts_df is None:
                 logs.warning("Vocab not yet loaded; doing so.")
                 self.load_or_prepare_vocab()
@@ -544,8 +544,8 @@ class DatasetStatisticsCacheClass:
         """
         logs.info("Doing text dset.")
         self.load_or_prepare_text_dset(save)
-        logs.info("Doing tokenized dataframe")
-        self.load_or_prepare_tokenized_df(save)
+        #logs.info("Doing tokenized dataframe")
+        #self.load_or_prepare_tokenized_df(save)
         logs.info("Doing dataset peek")
         self.load_or_prepare_dset_peek(save)
 
@@ -554,11 +554,12 @@ class DatasetStatisticsCacheClass:
             with open(self.dset_peek_json_fid, "r") as f:
                 self.dset_peek = json.load(f)["dset peek"]
         else:
-            if self.dset is None:
-                self.get_base_dataset()
-            self.dset_peek = self.dset[:100]
-            if save:
-                write_json({"dset peek": self.dset_peek}, self.dset_peek_json_fid)
+            if not self.live:
+                if self.dset is None:
+                    self.get_base_dataset()
+                self.dset_peek = self.dset[:100]
+                if save:
+                    write_json({"dset peek": self.dset_peek}, self.dset_peek_json_fid)
 
     def load_or_prepare_tokenized_df(self, save=True):
         if self.use_cache and exists(self.tokenized_df_fid):

data_measurements/streamlit_utils.py

@@ -20,7 +20,7 @@ import streamlit as st
 from st_aggrid import AgGrid, GridOptionsBuilder
 
 from .dataset_utils import HF_DESC_FIELD, HF_FEATURE_FIELD, HF_LABEL_FIELD
-
+st.set_option('deprecation.showPyplotGlobalUse', False)
 
 def sidebar_header():
     st.sidebar.markdown(
@@ -48,7 +48,10 @@ def sidebar_selection(ds_name_to_dict, column_id):
         )
         # choose a config to analyze
         ds_configs = ds_name_to_dict[ds_name]
-        config_names = list(ds_configs.keys())
+        if ds_name == "c4":
+            config_names = ['en','en.noblocklist','realnewslike']
+        else:
+            config_names = list(ds_configs.keys())
         config_name = st.selectbox(
             f"Choose configuration{column_id}:",
             config_names,
@@ -319,72 +322,75 @@ def expander_npmi_description(min_vocab):
 
 ### Finally, show Zipf stuff
 def expander_zipf(z, zipf_fig, column_id):
-    _ZIPF_CAPTION = """This shows how close the observed language is to an ideal
-    natural language distribution following [Zipf's law](https://en.wikipedia.org/wiki/Zipf%27s_law),
-    calculated by minimizing the [Kolmogorov-Smirnov (KS) statistic](https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test)."""
-
-    powerlaw_eq = r"""p(x) \propto x^{- \alpha}"""
-    zipf_summary = (
-        "The optimal alpha based on this dataset is: **"
-        + str(round(z.alpha, 2))
-        + "**, with a KS distance of: **"
-        + str(round(z.distance, 2))
-    )
-    zipf_summary += (
-        "**.  This was fit with a minimum rank value of: **"
-        + str(int(z.xmin))
-        + "**, which is the optimal rank *beyond which* the scaling regime of the power law fits best."
-    )
-
-    alpha_warning = "Your alpha value is a bit on the high side, which means that the distribution over words in this dataset is a bit unnatural. This could be due to non-language items throughout the dataset."
-    xmin_warning = "The minimum rank for this fit is a bit on the high side, which means that the frequencies of your most common words aren't distributed as would be expected by Zipf's law."
-    fit_results_table = pd.DataFrame.from_dict(
-        {
-            r"Alpha:": [str("%.2f" % z.alpha)],
-            "KS distance:": [str("%.2f" % z.distance)],
-            "Min rank:": [str("%s" % int(z.xmin))],
-        },
-        columns=["Results"],
-        orient="index",
-    )
-    fit_results_table.index.name = column_id
     with st.expander(
         f"Vocabulary Distribution{column_id}: Zipf's Law Fit", expanded=False
     ):
-        st.caption(
-            "Use this widget for the counts of different words in your dataset, measuring the difference between the observed count and the expected count under Zipf's law."
-        )
-        st.markdown(_ZIPF_CAPTION)
-        st.write(
-            """
-        A Zipfian distribution follows the power law: $p(x) \propto x^{-α}$
-with an ideal α value of 1."""
-        )
-        st.markdown(
-            "In general, an alpha greater than 2 or a minimum rank greater than 10 (take with a grain of salt) means that your distribution is relativaly _unnatural_ for natural language. This can be a sign of mixed artefacts in the dataset, such as HTML markup."
-        )
-        st.markdown(
-            "Below, you can see the counts of each word in your dataset vs. the expected number of counts following a Zipfian distribution."
-        )
-        st.markdown("-----")
-        st.write("### Here is your dataset's Zipf results:")
-        st.dataframe(fit_results_table)
-        st.write(zipf_summary)
-        # TODO: Nice UI version of the content in the comments.
-        # st.markdown("\nThe KS test p-value is < %.2f" % z.ks_test.pvalue)
-        # if z.ks_test.pvalue < 0.01:
-        #    st.markdown(
-        #        "\n Great news! Your data fits a powerlaw with a minimum KS " "distance of %.4f" % z.distance)
-        # else:
-        #    st.markdown("\n Sadly, your data does not fit a powerlaw. =(")
-        # st.markdown("Checking the goodness of fit of our observed distribution")
-        # st.markdown("to the hypothesized power law distribution")
-        # st.markdown("using a Kolmogorov–Smirnov (KS) test.")
-        st.plotly_chart(zipf_fig, use_container_width=True)
-        if z.alpha > 2:
-            st.markdown(alpha_warning)
-        if z.xmin > 5:
-            st.markdown(xmin_warning)
+        try:
+            _ZIPF_CAPTION = """This shows how close the observed language is to an ideal
+            natural language distribution following [Zipf's law](https://en.wikipedia.org/wiki/Zipf%27s_law),
+            calculated by minimizing the [Kolmogorov-Smirnov (KS) statistic](https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test)."""
+
+            powerlaw_eq = r"""p(x) \propto x^{- \alpha}"""
+            zipf_summary = (
+                    "The optimal alpha based on this dataset is: **"
+                    + str(round(z.alpha, 2))
+                    + "**, with a KS distance of: **"
+                    + str(round(z.distance, 2))
+            )
+            zipf_summary += (
+                    "**.  This was fit with a minimum rank value of: **"
+                    + str(int(z.xmin))
+                    + "**, which is the optimal rank *beyond which* the scaling regime of the power law fits best."
+            )
+
+            alpha_warning = "Your alpha value is a bit on the high side, which means that the distribution over words in this dataset is a bit unnatural. This could be due to non-language items throughout the dataset."
+            xmin_warning = "The minimum rank for this fit is a bit on the high side, which means that the frequencies of your most common words aren't distributed as would be expected by Zipf's law."
+            fit_results_table = pd.DataFrame.from_dict(
+                {
+                    r"Alpha:": [str("%.2f" % z.alpha)],
+                    "KS distance:": [str("%.2f" % z.distance)],
+                    "Min rank:": [str("%s" % int(z.xmin))],
+                },
+                columns=["Results"],
+                orient="index",
+            )
+            fit_results_table.index.name = column_id
+            st.caption(
+                "Use this widget for the counts of different words in your dataset, measuring the difference between the observed count and the expected count under Zipf's law."
+            )
+            st.markdown(_ZIPF_CAPTION)
+            st.write(
+                """
+            A Zipfian distribution follows the power law: $p(x) \propto x^{-α}$
+    with an ideal α value of 1."""
+            )
+            st.markdown(
+                "In general, an alpha greater than 2 or a minimum rank greater than 10 (take with a grain of salt) means that your distribution is relativaly _unnatural_ for natural language. This can be a sign of mixed artefacts in the dataset, such as HTML markup."
+            )
+            st.markdown(
+                "Below, you can see the counts of each word in your dataset vs. the expected number of counts following a Zipfian distribution."
+            )
+            st.markdown("-----")
+            st.write("### Here is your dataset's Zipf results:")
+            st.dataframe(fit_results_table)
+            st.write(zipf_summary)
+            # TODO: Nice UI version of the content in the comments.
+            # st.markdown("\nThe KS test p-value is < %.2f" % z.ks_test.pvalue)
+            # if z.ks_test.pvalue < 0.01:
+            #    st.markdown(
+            #        "\n Great news! Your data fits a powerlaw with a minimum KS " "distance of %.4f" % z.distance)
+            # else:
+            #    st.markdown("\n Sadly, your data does not fit a powerlaw. =(")
+            # st.markdown("Checking the goodness of fit of our observed distribution")
+            # st.markdown("to the hypothesized power law distribution")
+            # st.markdown("using a Kolmogorov–Smirnov (KS) test.")
+            st.plotly_chart(zipf_fig, use_container_width=True)
+            if z.alpha > 2:
+                st.markdown(alpha_warning)
+            if z.xmin > 5:
+                st.markdown(xmin_warning)
+        except:
+            st.write("Under construction!")
 
 
 ### Finally finally finally, show nPMI stuff.
@@ -427,17 +433,23 @@ def npmi_widget(npmi_stats, min_vocab, column_id):
 
 def npmi_show(paired_results):
     if paired_results.empty:
-        st.markdown("No words that co-occur enough times for results!  Or there's a 🐛.")
+        st.markdown("No words that co-occur enough times for results!  Or there's a 🐛.  Or we're still computing this one. 🤷")
     else:
         s = pd.DataFrame(paired_results.sort_values(by="npmi-bias", ascending=True))
         # s.columns=pd.MultiIndex.from_arrays([['npmi','npmi','npmi','count', 'count'],['bias','man','straight','man','straight']])
         s.index.name = "word"
         npmi_cols = s.filter(like="npmi").columns
         count_cols = s.filter(like="count").columns
+        if s.shape[0] > 10000:
+            bias_thres = max(abs(s["npmi-bias"][5000]), abs(s["npmi-bias"][-5000]))
+            print(f"filtering with bias threshold: {bias_thres}")
+            s_filtered = s[s["npmi-bias"].abs() > bias_thres]
+        else:
+            s_filtered = s
         # TODO: This is very different look than the duplicates table above. Should probably standardize.
         cm = sns.palplot(sns.diverging_palette(270, 36, s=99, l=48, n=16))
         out_df = (
-            s.style.background_gradient(subset=npmi_cols, cmap=cm)
+            s_filtered.style.background_gradient(subset=npmi_cols, cmap=cm)
             .format(subset=npmi_cols, formatter="{:,.3f}")
             .format(subset=count_cols, formatter=int)
             .set_properties(

requirements.txt

@@ -10,7 +10,7 @@ iso_639==0.4.5
 datasets==1.15.1
 powerlaw==1.5
 numpy==1.19.5
-pandas==1.3.0
+pandas==1.0.0
 dataclasses==0.6
 iso639==0.1.4
 python_igraph==0.9.6
@@ -23,4 +23,4 @@ numexpr==2.7.3
 scikit-learn~=0.24.2
 scipy~=1.7.3
 tqdm~=4.62.3
-pyarrow~=6.0.1
+pyarrow~=6.0.1