lalital commited on
Commit
a85d3a4
1 Parent(s): f074082

Add additional_special_tokens

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +7 -2
tokenizer_config.json CHANGED
@@ -5,5 +5,10 @@
5
  "sep_token": "</s>",
6
  "cls_token": "<s>",
7
  "pad_token": "<pad>",
8
- "mask_token": "<mask>"
9
- }
 
 
 
 
 
 
5
  "sep_token": "</s>",
6
  "cls_token": "<s>",
7
  "pad_token": "<pad>",
8
+ "mask_token": "<mask>",
9
+ "additional_special_tokens": [
10
+ "<s>NOTUSED",
11
+ "</s>NOTUSED",
12
+ "<_>"
13
+ ]
14
+ }