Add diversity penalty, pin tokenizers on older version

app.py

@@ -122,7 +122,7 @@ def main():
         "Num beam groups", min_value=1, max_value=10, value=1
     )
     length_penalty = st.sidebar.number_input(
-        "Length penalty", min_value=0.0, max_value=2.0, value=1.2, step=0.1
+        "Length penalty", min_value=0.0, max_value=2.0, value=1.0, step=0.1
     )
     diversity_penalty = st.sidebar.number_input(
         "Diversity penalty", min_value=0.0, max_value=2.0, value=0.1, step=0.1
@@ -136,7 +136,7 @@ and the [Huggingface text generation interface doc](https://huggingface.co/trans
         "num_beams": num_beams,
         "num_beam_groups": num_beam_groups,
         "diversity_penalty": diversity_penalty if num_beam_groups > 1 else 0.0,
-        "length_penalty": length_penalty,
+        "length_penalty": length_penalty if num_beams > 1 else 1.0,
         "early_stopping": True,
     }
 

generator.py

@@ -1,4 +1,3 @@
-import _thread
 import os
 import re
 

requirements.txt

@@ -1,11 +1,12 @@
-streamlit
-torch
-transformers
+streamlit~=1.25.0
+torch~=2.0.0
+transformers~=4.30.0
+tokenizers~=0.13.3
 langdetect
 psutil
-jax<0.4.0
-jaxlib<0.4.0
-chex>=0.1.4
-flax<0.6.0
+jax==0.4.13
+jaxlib==0.4.13
+chex
+flax
 sentencepiece
 nltk