Improve tokenizer backwards compatibility

added_tokens.json

@@ -1,102 +0,0 @@
-{
-  "<extra_id_0>": 32099,
-  "<extra_id_10>": 32089,
-  "<extra_id_11>": 32088,
-  "<extra_id_12>": 32087,
-  "<extra_id_13>": 32086,
-  "<extra_id_14>": 32085,
-  "<extra_id_15>": 32084,
-  "<extra_id_16>": 32083,
-  "<extra_id_17>": 32082,
-  "<extra_id_18>": 32081,
-  "<extra_id_19>": 32080,
-  "<extra_id_1>": 32098,
-  "<extra_id_20>": 32079,
-  "<extra_id_21>": 32078,
-  "<extra_id_22>": 32077,
-  "<extra_id_23>": 32076,
-  "<extra_id_24>": 32075,
-  "<extra_id_25>": 32074,
-  "<extra_id_26>": 32073,
-  "<extra_id_27>": 32072,
-  "<extra_id_28>": 32071,
-  "<extra_id_29>": 32070,
-  "<extra_id_2>": 32097,
-  "<extra_id_30>": 32069,
-  "<extra_id_31>": 32068,
-  "<extra_id_32>": 32067,
-  "<extra_id_33>": 32066,
-  "<extra_id_34>": 32065,
-  "<extra_id_35>": 32064,
-  "<extra_id_36>": 32063,
-  "<extra_id_37>": 32062,
-  "<extra_id_38>": 32061,
-  "<extra_id_39>": 32060,
-  "<extra_id_3>": 32096,
-  "<extra_id_40>": 32059,
-  "<extra_id_41>": 32058,
-  "<extra_id_42>": 32057,
-  "<extra_id_43>": 32056,
-  "<extra_id_44>": 32055,
-  "<extra_id_45>": 32054,
-  "<extra_id_46>": 32053,
-  "<extra_id_47>": 32052,
-  "<extra_id_48>": 32051,
-  "<extra_id_49>": 32050,
-  "<extra_id_4>": 32095,
-  "<extra_id_50>": 32049,
-  "<extra_id_51>": 32048,
-  "<extra_id_52>": 32047,
-  "<extra_id_53>": 32046,
-  "<extra_id_54>": 32045,
-  "<extra_id_55>": 32044,
-  "<extra_id_56>": 32043,
-  "<extra_id_57>": 32042,
-  "<extra_id_58>": 32041,
-  "<extra_id_59>": 32040,
-  "<extra_id_5>": 32094,
-  "<extra_id_60>": 32039,
-  "<extra_id_61>": 32038,
-  "<extra_id_62>": 32037,
-  "<extra_id_63>": 32036,
-  "<extra_id_64>": 32035,
-  "<extra_id_65>": 32034,
-  "<extra_id_66>": 32033,
-  "<extra_id_67>": 32032,
-  "<extra_id_68>": 32031,
-  "<extra_id_69>": 32030,
-  "<extra_id_6>": 32093,
-  "<extra_id_70>": 32029,
-  "<extra_id_71>": 32028,
-  "<extra_id_72>": 32027,
-  "<extra_id_73>": 32026,
-  "<extra_id_74>": 32025,
-  "<extra_id_75>": 32024,
-  "<extra_id_76>": 32023,
-  "<extra_id_77>": 32022,
-  "<extra_id_78>": 32021,
-  "<extra_id_79>": 32020,
-  "<extra_id_7>": 32092,
-  "<extra_id_80>": 32019,
-  "<extra_id_81>": 32018,
-  "<extra_id_82>": 32017,
-  "<extra_id_83>": 32016,
-  "<extra_id_84>": 32015,
-  "<extra_id_85>": 32014,
-  "<extra_id_86>": 32013,
-  "<extra_id_87>": 32012,
-  "<extra_id_88>": 32011,
-  "<extra_id_89>": 32010,
-  "<extra_id_8>": 32091,
-  "<extra_id_90>": 32009,
-  "<extra_id_91>": 32008,
-  "<extra_id_92>": 32007,
-  "<extra_id_93>": 32006,
-  "<extra_id_94>": 32005,
-  "<extra_id_95>": 32004,
-  "<extra_id_96>": 32003,
-  "<extra_id_97>": 32002,
-  "<extra_id_98>": 32001,
-  "<extra_id_99>": 32000,
-  "<extra_id_9>": 32090
-}

special_tokens_map.json

@@ -101,25 +101,7 @@
     "<extra_id_98>",
     "<extra_id_99>"
   ],
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<pad>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
 }

tokenizer_config.json

@@ -927,12 +927,9 @@
     "<extra_id_98>",
     "<extra_id_99>"
   ],
-  "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
   "extra_ids": 100,
-  "legacy": true,
   "pad_token": "<pad>",
-  "sp_model_kwargs": {},
   "tokenizer_class": "T5Tokenizer",
   "unk_token": "<unk>"
 }