Update app.py

app.py

@@ -5,9 +5,9 @@ from typing import Iterator
 import gradio as gr
 import spaces
 import torch
-from transformers import AutoModelForCausalLM, GemmaTokenizerFast, TextIteratorStreamer
+from transformers import BitsAndBytesConfig, AutoModelForCausalLM, GemmaTokenizerFast, TextIteratorStreamer
 
-huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
+huggingface_token = os.getenv('read_access')
 
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024
@@ -17,10 +17,13 @@ device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
 model_id = "google/gemma-2-9b-it"
 tokenizer = GemmaTokenizerFast.from_pretrained(model_id, token = huggingface_token)
+
+quantization = BitsAndBytesConfig(load_in_8bit= True)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     device_map="auto",
     torch_dtype=torch.bfloat16,
+    quantization_config=quantization,
     token = huggingface_token
 )
 model.config.sliding_window = 4096