satpalsr commited on
Commit
f7fdf66
1 Parent(s): dd6075a

Upload tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +3 -3
tokenizer.json CHANGED
@@ -23,7 +23,7 @@
23
  },
24
  {
25
  "id": 128002,
26
- "content": "<|reserved_special_token_0|>",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
@@ -32,7 +32,7 @@
32
  },
33
  {
34
  "id": 128003,
35
- "content": "<|reserved_special_token_1|>",
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
@@ -410560,4 +410560,4 @@
410560
  "éĶ ¦"
410561
  ]
410562
  }
410563
- }
 
23
  },
24
  {
25
  "id": 128002,
26
+ "content": "<|im_start|>",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
 
32
  },
33
  {
34
  "id": 128003,
35
+ "content": "<|im_end|>",
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
 
410560
  "éĶ ¦"
410561
  ]
410562
  }
410563
+ }