DrNicefellow
commited on
Commit
•
3a380af
1
Parent(s):
1870271
Update tokenizer_config.json
Browse files- tokenizer_config.json +0 -1
tokenizer_config.json
CHANGED
@@ -15,6 +15,5 @@
|
|
15 |
"eos_token": "<|endoftext|>",
|
16 |
"model_max_length": 1024,
|
17 |
"tokenizer_class": "GPT2Tokenizer",
|
18 |
-
"chat_template": "{%- for message in messages -%} {%- if message['role'] == 'system' -%} {%- set ns.found = true -%} {%- endif -%} {%- endfor -%} {%- if not ns.found -%} {{- '' + 'A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user\'s questions.' + '\n\n' -}} {%- endif -%} {%- for message in messages %} {%- if message['role'] == 'system' -%} {{- '' + message['content'] + '\n\n' -}} {%- else -%} {%- if message['role'] == 'user' -%} {{-'USER: ' + message['content'] + '\n'-}} {%- else -%} {{-'ASSISTANT: ' + message['content'] + '</s>\n' -}} {%- endif -%} {%- endif -%} {%- endfor -%} {%- if add_generation_prompt -%} {{-'ASSISTANT:'-}} {%- endif -%}",
|
19 |
"unk_token": "<|endoftext|>"
|
20 |
}
|
|
|
15 |
"eos_token": "<|endoftext|>",
|
16 |
"model_max_length": 1024,
|
17 |
"tokenizer_class": "GPT2Tokenizer",
|
|
|
18 |
"unk_token": "<|endoftext|>"
|
19 |
}
|