test

app.py

@@ -8,13 +8,14 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 # Load the tokenizer and model
-tokenizer = AutoTokenizer.from_pretrained("berkeley-nest/Starling-LM-7B-alpha").to(device)
+tokenizer = AutoTokenizer.from_pretrained("berkeley-nest/Starling-LM-7B-alpha")
 model = AutoModelForCausalLM.from_pretrained("berkeley-nest/Starling-LM-7B-alpha").to(device)
+model.eval()  # Set the model to evaluation mode
 
 @spaces.GPU
 def generate_response(user_input, chat_history):
     try:
-        prompt = "GPT4 Correct User: " + user_input + "<|end_of_turn|>" + "GPT4 Correct Assistant: "
+        prompt = "GPT4 Correct User: " + user_input + "GPT4 Correct Assistant: "
         if chat_history:
             prompt = chat_history[-1024:] + prompt  # Keep last 1024 tokens of history
         
@@ -49,4 +50,4 @@ with gr.Blocks(gr.themes.Soft()) as app:
     send.click(generate_response, inputs=[user_input, chat_history], outputs=[chatbot, chat_history])
     clear.click(clear_chat, outputs=[chatbot, chat_history])
 
-app.launch()
+app.launch()