Update tools/llama/generate.py

tools/llama/generate.py

@@ -22,6 +22,7 @@ torch.cuda.is_available = lambda: False
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 torch._inductor.config.coordinate_descent_tuning = True
 torch._inductor.config.triton.unique_kernel_names = True
+device = torch.device('cpu')
 
 if hasattr(torch._inductor.config, "fx_graph_cache"):
     # Experimental feature to reduce compilation times, will be on by default in future