lalital commited on
Commit
3815314
1 Parent(s): 1049ef5

Add additional_special_tokens

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +6 -1
tokenizer_config.json CHANGED
@@ -5,5 +5,10 @@
5
  "sep_token": "</s>",
6
  "cls_token": "<s>",
7
  "pad_token": "<pad>",
8
- "mask_token": "<mask>"
 
 
 
 
 
9
  }
 
5
  "sep_token": "</s>",
6
  "cls_token": "<s>",
7
  "pad_token": "<pad>",
8
+ "mask_token": "<mask>",
9
+ "additional_special_tokens": [
10
+ "<s>NOTUSED",
11
+ "</s>NOTUSED",
12
+ "<_>"
13
+ ]
14
  }