# palmyra-4-oasis | |
![image/jpeg](https://cdn-uploads.huggingface.co/production/uploads/621d6f532165dc431641e438/_Dt7ota295BoQtQ9cake5.jpeg) | |
```py | |
import torch | |
from transformers import AutoTokenizer, AutoModelForCausalLM | |
model_id = "Writer/palmyra-4-oasis" | |
tokenizer = AutoTokenizer.from_pretrained(model_id) | |
model = AutoModelForCausalLM.from_pretrained( | |
model_id, | |
torch_dtype=torch.float16, | |
device_map="auto", | |
attn_implementation="flash_attention_2", | |
) | |
messages = [ | |
{ | |
"role": "user", | |
"content": "who developed you?", | |
}, | |
] | |
input_ids = tokenizer.apply_chat_template( | |
messages, tokenize=True, add_generation_prompt=True, return_tensors="pt" | |
) | |
gen_conf = { | |
"max_new_tokens": 1024, | |
"eos_token_id": tokenizer.eos_token_id, | |
"temperature": 0.0, | |
"top_p": 0.9, | |
} | |
with torch.inference_mode(): | |
output_id = model.generate(input_ids, **gen_conf) | |
output_text = tokenizer.decode(output_id[0][input_ids.shape[1] :]) | |
print(output_text) | |
``` |