jon-tow commited on
Commit
00e549c
1 Parent(s): c4e0a2b

tmpfix(tokenizer_config): force `GPT2TokenizerFast`

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -306,6 +306,6 @@
306
  "eos_token": "<|endoftext|>",
307
  "model_max_length": 1000000000000000019884624838656,
308
  "pad_token": "<|endoftext|>",
309
- "tokenizer_class": "GPT2Tokenizer",
310
  "unk_token": "<|endoftext|>"
311
  }
 
306
  "eos_token": "<|endoftext|>",
307
  "model_max_length": 1000000000000000019884624838656,
308
  "pad_token": "<|endoftext|>",
309
+ "tokenizer_class": "GPT2TokenizerFast",
310
  "unk_token": "<|endoftext|>"
311
  }