Hugging Face's logo

Polygl0t
/
Tucano2-0.6B-Base

Text Generation
Transformers
Safetensors
Portuguese
llama
text-generation-inference
Eval Results (legacy)
Model card Files
xet
 Community
nicholasKluge commited on
Commit
ad1616e
·
verified ·
1 Parent(s): d6df073

Upload 3 files

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +0 -44
  2. tokenizer.json +42 -42
  3. tokenizer_config.json +45 -89
special_tokens_map.json CHANGED
@@ -1,48 +1,4 @@
1
  {
2
- "additional_special_tokens": [
3
- "<tools>",
4
- "</tools>",
5
- "<tool_call>",
6
- "</tool_call>",
7
- "<tool_response>",
8
- "</tool_response>",
9
- "<think>",
10
- "</think>",
11
- "<answer>",
12
- "</answer>",
13
- "<context>",
14
- "</context>",
15
- "<|fim_prefix|>",
16
- "<|fim_suffix|>",
17
- "<|fim_middle|>",
18
- "<|repo_name|>",
19
- "<|image|>",
20
- "<|image_pad|>",
21
- "<|image_placeholder|>",
22
- " ",
23
- " ",
24
- " ",
25
- " ",
26
- " ",
27
- " ",
28
- " ",
29
- " ",
30
- " ",
31
- " ",
32
- " ",
33
- " ",
34
- " ",
35
- " ",
36
- " ",
37
- " ",
38
- " ",
39
- " ",
40
- " ",
41
- " ",
42
- " ",
43
- " ",
44
- " "
45
- ],
46
  "bos_token": {
47
  "content": "<|im_start|>",
48
  "lstrip": false,
 
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "bos_token": {
3
  "content": "<|im_start|>",
4
  "lstrip": false,
tokenizer.json CHANGED
@@ -46,7 +46,7 @@
46
  "lstrip": false,
47
  "rstrip": false,
48
  "normalized": false,
49
- "special": true
50
  },
51
  {
52
  "id": 49111,
@@ -55,7 +55,7 @@
55
  "lstrip": false,
56
  "rstrip": false,
57
  "normalized": false,
58
- "special": true
59
  },
60
  {
61
  "id": 49112,
@@ -64,7 +64,7 @@
64
  "lstrip": false,
65
  "rstrip": false,
66
  "normalized": false,
67
- "special": true
68
  },
69
  {
70
  "id": 49113,
@@ -73,7 +73,7 @@
73
  "lstrip": false,
74
  "rstrip": false,
75
  "normalized": false,
76
- "special": true
77
  },
78
  {
79
  "id": 49114,
@@ -82,7 +82,7 @@
82
  "lstrip": false,
83
  "rstrip": false,
84
  "normalized": false,
85
- "special": true
86
  },
87
  {
88
  "id": 49115,
@@ -91,7 +91,7 @@
91
  "lstrip": false,
92
  "rstrip": false,
93
  "normalized": false,
94
- "special": true
95
  },
96
  {
97
  "id": 49116,
@@ -100,7 +100,7 @@
100
  "lstrip": false,
101
  "rstrip": false,
102
  "normalized": false,
103
- "special": true
104
  },
105
  {
106
  "id": 49117,
@@ -109,7 +109,7 @@
109
  "lstrip": false,
110
  "rstrip": false,
111
  "normalized": false,
112
- "special": true
113
  },
114
  {
115
  "id": 49118,
@@ -118,7 +118,7 @@
118
  "lstrip": false,
119
  "rstrip": false,
120
  "normalized": false,
121
- "special": true
122
  },
123
  {
124
  "id": 49119,
@@ -127,7 +127,7 @@
127
  "lstrip": false,
128
  "rstrip": false,
129
  "normalized": false,
130
- "special": true
131
  },
132
  {
133
  "id": 49120,
@@ -136,7 +136,7 @@
136
  "lstrip": false,
137
  "rstrip": false,
138
  "normalized": false,
139
- "special": true
140
  },
141
  {
142
  "id": 49121,
@@ -145,7 +145,7 @@
145
  "lstrip": false,
146
  "rstrip": false,
147
  "normalized": false,
148
- "special": true
149
  },
150
  {
151
  "id": 49122,
@@ -154,7 +154,7 @@
154
  "lstrip": false,
155
  "rstrip": false,
156
  "normalized": false,
157
- "special": true
158
  },
159
  {
160
  "id": 49123,
@@ -163,7 +163,7 @@
163
  "lstrip": false,
164
  "rstrip": false,
165
  "normalized": false,
166
- "special": true
167
  },
168
  {
169
  "id": 49124,
@@ -172,7 +172,7 @@
172
  "lstrip": false,
173
  "rstrip": false,
174
  "normalized": false,
175
- "special": true
176
  },
177
  {
178
  "id": 49125,
@@ -181,7 +181,7 @@
181
  "lstrip": false,
182
  "rstrip": false,
183
  "normalized": false,
184
- "special": true
185
  },
186
  {
187
  "id": 49126,
@@ -190,7 +190,7 @@
190
  "lstrip": false,
191
  "rstrip": false,
192
  "normalized": false,
193
- "special": true
194
  },
195
  {
196
  "id": 49127,
@@ -199,7 +199,7 @@
199
  "lstrip": false,
200
  "rstrip": false,
201
  "normalized": false,
202
- "special": true
203
  },
204
  {
205
  "id": 49128,
@@ -208,7 +208,7 @@
208
  "lstrip": false,
209
  "rstrip": false,
210
  "normalized": false,
211
- "special": true
212
  },
213
  {
214
  "id": 49129,
@@ -217,7 +217,7 @@
217
  "lstrip": false,
218
  "rstrip": false,
219
  "normalized": false,
220
- "special": true
221
  },
222
  {
223
  "id": 49130,
@@ -226,7 +226,7 @@
226
  "lstrip": false,
227
  "rstrip": false,
228
  "normalized": false,
229
- "special": true
230
  },
231
  {
232
  "id": 49131,
@@ -235,7 +235,7 @@
235
  "lstrip": false,
236
  "rstrip": false,
237
  "normalized": false,
238
- "special": true
239
  },
240
  {
241
  "id": 49132,
@@ -244,7 +244,7 @@
244
  "lstrip": false,
245
  "rstrip": false,
246
  "normalized": false,
247
- "special": true
248
  },
249
  {
250
  "id": 49133,
@@ -253,7 +253,7 @@
253
  "lstrip": false,
254
  "rstrip": false,
255
  "normalized": false,
256
- "special": true
257
  },
258
  {
259
  "id": 49134,
@@ -262,7 +262,7 @@
262
  "lstrip": false,
263
  "rstrip": false,
264
  "normalized": false,
265
- "special": true
266
  },
267
  {
268
  "id": 49135,
@@ -271,7 +271,7 @@
271
  "lstrip": false,
272
  "rstrip": false,
273
  "normalized": false,
274
- "special": true
275
  },
276
  {
277
  "id": 49136,
@@ -280,7 +280,7 @@
280
  "lstrip": false,
281
  "rstrip": false,
282
  "normalized": false,
283
- "special": true
284
  },
285
  {
286
  "id": 49137,
@@ -289,7 +289,7 @@
289
  "lstrip": false,
290
  "rstrip": false,
291
  "normalized": false,
292
- "special": true
293
  },
294
  {
295
  "id": 49138,
@@ -298,7 +298,7 @@
298
  "lstrip": false,
299
  "rstrip": false,
300
  "normalized": false,
301
- "special": true
302
  },
303
  {
304
  "id": 49139,
@@ -307,7 +307,7 @@
307
  "lstrip": false,
308
  "rstrip": false,
309
  "normalized": false,
310
- "special": true
311
  },
312
  {
313
  "id": 49140,
@@ -316,7 +316,7 @@
316
  "lstrip": false,
317
  "rstrip": false,
318
  "normalized": false,
319
- "special": true
320
  },
321
  {
322
  "id": 49141,
@@ -325,7 +325,7 @@
325
  "lstrip": false,
326
  "rstrip": false,
327
  "normalized": false,
328
- "special": true
329
  },
330
  {
331
  "id": 49142,
@@ -334,7 +334,7 @@
334
  "lstrip": false,
335
  "rstrip": false,
336
  "normalized": false,
337
- "special": true
338
  },
339
  {
340
  "id": 49143,
@@ -343,7 +343,7 @@
343
  "lstrip": false,
344
  "rstrip": false,
345
  "normalized": false,
346
- "special": true
347
  },
348
  {
349
  "id": 49144,
@@ -352,7 +352,7 @@
352
  "lstrip": false,
353
  "rstrip": false,
354
  "normalized": false,
355
- "special": true
356
  },
357
  {
358
  "id": 49145,
@@ -361,7 +361,7 @@
361
  "lstrip": false,
362
  "rstrip": false,
363
  "normalized": false,
364
- "special": true
365
  },
366
  {
367
  "id": 49146,
@@ -370,7 +370,7 @@
370
  "lstrip": false,
371
  "rstrip": false,
372
  "normalized": false,
373
- "special": true
374
  },
375
  {
376
  "id": 49147,
@@ -379,7 +379,7 @@
379
  "lstrip": false,
380
  "rstrip": false,
381
  "normalized": false,
382
- "special": true
383
  },
384
  {
385
  "id": 49148,
@@ -388,7 +388,7 @@
388
  "lstrip": false,
389
  "rstrip": false,
390
  "normalized": false,
391
- "special": true
392
  },
393
  {
394
  "id": 49149,
@@ -397,7 +397,7 @@
397
  "lstrip": false,
398
  "rstrip": false,
399
  "normalized": false,
400
- "special": true
401
  },
402
  {
403
  "id": 49150,
@@ -406,7 +406,7 @@
406
  "lstrip": false,
407
  "rstrip": false,
408
  "normalized": false,
409
- "special": true
410
  },
411
  {
412
  "id": 49151,
@@ -415,7 +415,7 @@
415
  "lstrip": false,
416
  "rstrip": false,
417
  "normalized": false,
418
- "special": true
419
  }
420
  ],
421
  "normalizer": null,
 
46
  "lstrip": false,
47
  "rstrip": false,
48
  "normalized": false,
49
+ "special": false
50
  },
51
  {
52
  "id": 49111,
 
55
  "lstrip": false,
56
  "rstrip": false,
57
  "normalized": false,
58
+ "special": false
59
  },
60
  {
61
  "id": 49112,
 
64
  "lstrip": false,
65
  "rstrip": false,
66
  "normalized": false,
67
+ "special": false
68
  },
69
  {
70
  "id": 49113,
 
73
  "lstrip": false,
74
  "rstrip": false,
75
  "normalized": false,
76
+ "special": false
77
  },
78
  {
79
  "id": 49114,
 
82
  "lstrip": false,
83
  "rstrip": false,
84
  "normalized": false,
85
+ "special": false
86
  },
87
  {
88
  "id": 49115,
 
91
  "lstrip": false,
92
  "rstrip": false,
93
  "normalized": false,
94
+ "special": false
95
  },
96
  {
97
  "id": 49116,
 
100
  "lstrip": false,
101
  "rstrip": false,
102
  "normalized": false,
103
+ "special": false
104
  },
105
  {
106
  "id": 49117,
 
109
  "lstrip": false,
110
  "rstrip": false,
111
  "normalized": false,
112
+ "special": false
113
  },
114
  {
115
  "id": 49118,
 
118
  "lstrip": false,
119
  "rstrip": false,
120
  "normalized": false,
121
+ "special": false
122
  },
123
  {
124
  "id": 49119,
 
127
  "lstrip": false,
128
  "rstrip": false,
129
  "normalized": false,
130
+ "special": false
131
  },
132
  {
133
  "id": 49120,
 
136
  "lstrip": false,
137
  "rstrip": false,
138
  "normalized": false,
139
+ "special": false
140
  },
141
  {
142
  "id": 49121,
 
145
  "lstrip": false,
146
  "rstrip": false,
147
  "normalized": false,
148
+ "special": false
149
  },
150
  {
151
  "id": 49122,
 
154
  "lstrip": false,
155
  "rstrip": false,
156
  "normalized": false,
157
+ "special": false
158
  },
159
  {
160
  "id": 49123,
 
163
  "lstrip": false,
164
  "rstrip": false,
165
  "normalized": false,
166
+ "special": false
167
  },
168
  {
169
  "id": 49124,
 
172
  "lstrip": false,
173
  "rstrip": false,
174
  "normalized": false,
175
+ "special": false
176
  },
177
  {
178
  "id": 49125,
 
181
  "lstrip": false,
182
  "rstrip": false,
183
  "normalized": false,
184
+ "special": false
185
  },
186
  {
187
  "id": 49126,
 
190
  "lstrip": false,
191
  "rstrip": false,
192
  "normalized": false,
193
+ "special": false
194
  },
195
  {
196
  "id": 49127,
 
199
  "lstrip": false,
200
  "rstrip": false,
201
  "normalized": false,
202
+ "special": false
203
  },
204
  {
205
  "id": 49128,
 
208
  "lstrip": false,
209
  "rstrip": false,
210
  "normalized": false,
211
+ "special": false
212
  },
213
  {
214
  "id": 49129,
 
217
  "lstrip": false,
218
  "rstrip": false,
219
  "normalized": false,
220
+ "special": false
221
  },
222
  {
223
  "id": 49130,
 
226
  "lstrip": false,
227
  "rstrip": false,
228
  "normalized": false,
229
+ "special": false
230
  },
231
  {
232
  "id": 49131,
 
235
  "lstrip": false,
236
  "rstrip": false,
237
  "normalized": false,
238
+ "special": false
239
  },
240
  {
241
  "id": 49132,
 
244
  "lstrip": false,
245
  "rstrip": false,
246
  "normalized": false,
247
+ "special": false
248
  },
249
  {
250
  "id": 49133,
 
253
  "lstrip": false,
254
  "rstrip": false,
255
  "normalized": false,
256
+ "special": false
257
  },
258
  {
259
  "id": 49134,
 
262
  "lstrip": false,
263
  "rstrip": false,
264
  "normalized": false,
265
+ "special": false
266
  },
267
  {
268
  "id": 49135,
 
271
  "lstrip": false,
272
  "rstrip": false,
273
  "normalized": false,
274
+ "special": false
275
  },
276
  {
277
  "id": 49136,
 
280
  "lstrip": false,
281
  "rstrip": false,
282
  "normalized": false,
283
+ "special": false
284
  },
285
  {
286
  "id": 49137,
 
289
  "lstrip": false,
290
  "rstrip": false,
291
  "normalized": false,
292
+ "special": false
293
  },
294
  {
295
  "id": 49138,
 
298
  "lstrip": false,
299
  "rstrip": false,
300
  "normalized": false,
301
+ "special": false
302
  },
303
  {
304
  "id": 49139,
 
307
  "lstrip": false,
308
  "rstrip": false,
309
  "normalized": false,
310
+ "special": false
311
  },
312
  {
313
  "id": 49140,
 
316
  "lstrip": false,
317
  "rstrip": false,
318
  "normalized": false,
319
+ "special": false
320
  },
321
  {
322
  "id": 49141,
 
325
  "lstrip": false,
326
  "rstrip": false,
327
  "normalized": false,
328
+ "special": false
329
  },
330
  {
331
  "id": 49142,
 
334
  "lstrip": false,
335
  "rstrip": false,
336
  "normalized": false,
337
+ "special": false
338
  },
339
  {
340
  "id": 49143,
 
343
  "lstrip": false,
344
  "rstrip": false,
345
  "normalized": false,
346
+ "special": false
347
  },
348
  {
349
  "id": 49144,
 
352
  "lstrip": false,
353
  "rstrip": false,
354
  "normalized": false,
355
+ "special": false
356
  },
357
  {
358
  "id": 49145,
 
361
  "lstrip": false,
362
  "rstrip": false,
363
  "normalized": false,
364
+ "special": false
365
  },
366
  {
367
  "id": 49146,
 
370
  "lstrip": false,
371
  "rstrip": false,
372
  "normalized": false,
373
+ "special": false
374
  },
375
  {
376
  "id": 49147,
 
379
  "lstrip": false,
380
  "rstrip": false,
381
  "normalized": false,
382
+ "special": false
383
  },
384
  {
385
  "id": 49148,
 
388
  "lstrip": false,
389
  "rstrip": false,
390
  "normalized": false,
391
+ "special": false
392
  },
393
  {
394
  "id": 49149,
 
397
  "lstrip": false,
398
  "rstrip": false,
399
  "normalized": false,
400
+ "special": false
401
  },
402
  {
403
  "id": 49150,
 
406
  "lstrip": false,
407
  "rstrip": false,
408
  "normalized": false,
409
+ "special": false
410
  },
411
  {
412
  "id": 49151,
 
415
  "lstrip": false,
416
  "rstrip": false,
417
  "normalized": false,
418
+ "special": false
419
  }
420
  ],
421
  "normalizer": null,
tokenizer_config.json CHANGED
@@ -41,7 +41,7 @@
41
  "normalized": false,
42
  "rstrip": false,
43
  "single_word": false,
44
- "special": true
45
  },
46
  "49111": {
47
  "content": "</tools>",
@@ -49,7 +49,7 @@
49
  "normalized": false,
50
  "rstrip": false,
51
  "single_word": false,
52
- "special": true
53
  },
54
  "49112": {
55
  "content": "<tool_call>",
@@ -57,7 +57,7 @@
57
  "normalized": false,
58
  "rstrip": false,
59
  "single_word": false,
60
- "special": true
61
  },
62
  "49113": {
63
  "content": "</tool_call>",
@@ -65,7 +65,7 @@
65
  "normalized": false,
66
  "rstrip": false,
67
  "single_word": false,
68
- "special": true
69
  },
70
  "49114": {
71
  "content": "<tool_response>",
@@ -73,7 +73,7 @@
73
  "normalized": false,
74
  "rstrip": false,
75
  "single_word": false,
76
- "special": true
77
  },
78
  "49115": {
79
  "content": "</tool_response>",
@@ -81,7 +81,7 @@
81
  "normalized": false,
82
  "rstrip": false,
83
  "single_word": false,
84
- "special": true
85
  },
86
  "49116": {
87
  "content": "<think>",
@@ -89,7 +89,7 @@
89
  "normalized": false,
90
  "rstrip": false,
91
  "single_word": false,
92
- "special": true
93
  },
94
  "49117": {
95
  "content": "</think>",
@@ -97,7 +97,7 @@
97
  "normalized": false,
98
  "rstrip": false,
99
  "single_word": false,
100
- "special": true
101
  },
102
  "49118": {
103
  "content": "<answer>",
@@ -105,7 +105,7 @@
105
  "normalized": false,
106
  "rstrip": false,
107
  "single_word": false,
108
- "special": true
109
  },
110
  "49119": {
111
  "content": "</answer>",
@@ -113,7 +113,7 @@
113
  "normalized": false,
114
  "rstrip": false,
115
  "single_word": false,
116
- "special": true
117
  },
118
  "49120": {
119
  "content": "<context>",
@@ -121,7 +121,7 @@
121
  "normalized": false,
122
  "rstrip": false,
123
  "single_word": false,
124
- "special": true
125
  },
126
  "49121": {
127
  "content": "</context>",
@@ -129,7 +129,7 @@
129
  "normalized": false,
130
  "rstrip": false,
131
  "single_word": false,
132
- "special": true
133
  },
134
  "49122": {
135
  "content": "<|fim_prefix|>",
@@ -137,7 +137,7 @@
137
  "normalized": false,
138
  "rstrip": false,
139
  "single_word": false,
140
- "special": true
141
  },
142
  "49123": {
143
  "content": "<|fim_suffix|>",
@@ -145,7 +145,7 @@
145
  "normalized": false,
146
  "rstrip": false,
147
  "single_word": false,
148
- "special": true
149
  },
150
  "49124": {
151
  "content": "<|fim_middle|>",
@@ -153,7 +153,7 @@
153
  "normalized": false,
154
  "rstrip": false,
155
  "single_word": false,
156
- "special": true
157
  },
158
  "49125": {
159
  "content": "<|repo_name|>",
@@ -161,7 +161,7 @@
161
  "normalized": false,
162
  "rstrip": false,
163
  "single_word": false,
164
- "special": true
165
  },
166
  "49126": {
167
  "content": "<|image|>",
@@ -169,7 +169,7 @@
169
  "normalized": false,
170
  "rstrip": false,
171
  "single_word": false,
172
- "special": true
173
  },
174
  "49127": {
175
  "content": "<|image_pad|>",
@@ -177,7 +177,7 @@
177
  "normalized": false,
178
  "rstrip": false,
179
  "single_word": false,
180
- "special": true
181
  },
182
  "49128": {
183
  "content": "<|image_placeholder|>",
@@ -185,7 +185,7 @@
185
  "normalized": false,
186
  "rstrip": false,
187
  "single_word": false,
188
- "special": true
189
  },
190
  "49129": {
191
  "content": " ",
@@ -193,7 +193,7 @@
193
  "normalized": false,
194
  "rstrip": false,
195
  "single_word": false,
196
- "special": true
197
  },
198
  "49130": {
199
  "content": " ",
@@ -201,7 +201,7 @@
201
  "normalized": false,
202
  "rstrip": false,
203
  "single_word": false,
204
- "special": true
205
  },
206
  "49131": {
207
  "content": " ",
@@ -209,7 +209,7 @@
209
  "normalized": false,
210
  "rstrip": false,
211
  "single_word": false,
212
- "special": true
213
  },
214
  "49132": {
215
  "content": " ",
@@ -217,7 +217,7 @@
217
  "normalized": false,
218
  "rstrip": false,
219
  "single_word": false,
220
- "special": true
221
  },
222
  "49133": {
223
  "content": " ",
@@ -225,7 +225,7 @@
225
  "normalized": false,
226
  "rstrip": false,
227
  "single_word": false,
228
- "special": true
229
  },
230
  "49134": {
231
  "content": " ",
@@ -233,7 +233,7 @@
233
  "normalized": false,
234
  "rstrip": false,
235
  "single_word": false,
236
- "special": true
237
  },
238
  "49135": {
239
  "content": " ",
@@ -241,7 +241,7 @@
241
  "normalized": false,
242
  "rstrip": false,
243
  "single_word": false,
244
- "special": true
245
  },
246
  "49136": {
247
  "content": " ",
@@ -249,7 +249,7 @@
249
  "normalized": false,
250
  "rstrip": false,
251
  "single_word": false,
252
- "special": true
253
  },
254
  "49137": {
255
  "content": " ",
@@ -257,7 +257,7 @@
257
  "normalized": false,
258
  "rstrip": false,
259
  "single_word": false,
260
- "special": true
261
  },
262
  "49138": {
263
  "content": " ",
@@ -265,7 +265,7 @@
265
  "normalized": false,
266
  "rstrip": false,
267
  "single_word": false,
268
- "special": true
269
  },
270
  "49139": {
271
  "content": " ",
@@ -273,7 +273,7 @@
273
  "normalized": false,
274
  "rstrip": false,
275
  "single_word": false,
276
- "special": true
277
  },
278
  "49140": {
279
  "content": " ",
@@ -281,7 +281,7 @@
281
  "normalized": false,
282
  "rstrip": false,
283
  "single_word": false,
284
- "special": true
285
  },
286
  "49141": {
287
  "content": " ",
@@ -289,7 +289,7 @@
289
  "normalized": false,
290
  "rstrip": false,
291
  "single_word": false,
292
- "special": true
293
  },
294
  "49142": {
295
  "content": " ",
@@ -297,7 +297,7 @@
297
  "normalized": false,
298
  "rstrip": false,
299
  "single_word": false,
300
- "special": true
301
  },
302
  "49143": {
303
  "content": " ",
@@ -305,7 +305,7 @@
305
  "normalized": false,
306
  "rstrip": false,
307
  "single_word": false,
308
- "special": true
309
  },
310
  "49144": {
311
  "content": " ",
@@ -313,7 +313,7 @@
313
  "normalized": false,
314
  "rstrip": false,
315
  "single_word": false,
316
- "special": true
317
  },
318
  "49145": {
319
  "content": " ",
@@ -321,7 +321,7 @@
321
  "normalized": false,
322
  "rstrip": false,
323
  "single_word": false,
324
- "special": true
325
  },
326
  "49146": {
327
  "content": " ",
@@ -329,7 +329,7 @@
329
  "normalized": false,
330
  "rstrip": false,
331
  "single_word": false,
332
- "special": true
333
  },
334
  "49147": {
335
  "content": " ",
@@ -337,7 +337,7 @@
337
  "normalized": false,
338
  "rstrip": false,
339
  "single_word": false,
340
- "special": true
341
  },
342
  "49148": {
343
  "content": " ",
@@ -353,7 +353,7 @@
353
  "normalized": false,
354
  "rstrip": false,
355
  "single_word": false,
356
- "special": true
357
  },
358
  "49150": {
359
  "content": " ",
@@ -361,7 +361,7 @@
361
  "normalized": false,
362
  "rstrip": false,
363
  "single_word": false,
364
- "special": true
365
  },
366
  "49151": {
367
  "content": " ",
@@ -369,53 +369,9 @@
369
  "normalized": false,
370
  "rstrip": false,
371
  "single_word": false,
372
- "special": true
373
  }
374
  },
375
- "additional_special_tokens": [
376
- "<tools>",
377
- "</tools>",
378
- "<tool_call>",
379
- "</tool_call>",
380
- "<tool_response>",
381
- "</tool_response>",
382
- "<think>",
383
- "</think>",
384
- "<answer>",
385
- "</answer>",
386
- "<context>",
387
- "</context>",
388
- "<|fim_prefix|>",
389
- "<|fim_suffix|>",
390
- "<|fim_middle|>",
391
- "<|repo_name|>",
392
- "<|image|>",
393
- "<|image_pad|>",
394
- "<|image_placeholder|>",
395
- " ",
396
- " ",
397
- " ",
398
- " ",
399
- " ",
400
- " ",
401
- " ",
402
- " ",
403
- " ",
404
- " ",
405
- " ",
406
- " ",
407
- " ",
408
- " ",
409
- " ",
410
- " ",
411
- " ",
412
- " ",
413
- " ",
414
- " ",
415
- " ",
416
- " ",
417
- " "
418
- ],
419
  "bos_token": "<|im_start|>",
420
  "bos_token_id": 1,
421
  "clean_up_tokenization_spaces": false,
@@ -423,6 +379,10 @@
423
  "eos_token_id": 2,
424
  "extra_special_tokens": {},
425
  "legacy": false,
 
 
 
 
426
  "model_max_length": 4096,
427
  "pad_token": "<|pad|>",
428
  "pad_token_id": 49109,
@@ -430,10 +390,6 @@
430
  "sp_model_kwargs": {},
431
  "spaces_between_special_tokens": false,
432
  "tokenizer_class": "PreTrainedTokenizerFast",
433
- "model_input_names": [
434
- "input_ids",
435
- "attention_mask"
436
- ],
437
  "truncation_side": "right",
438
  "unk_token": "<|unk|>",
439
  "unk_token_id": 0,
 
41
  "normalized": false,
42
  "rstrip": false,
43
  "single_word": false,
44
+ "special": false
45
  },
46
  "49111": {
47
  "content": "</tools>",
 
49
  "normalized": false,
50
  "rstrip": false,
51
  "single_word": false,
52
+ "special": false
53
  },
54
  "49112": {
55
  "content": "<tool_call>",
 
57
  "normalized": false,
58
  "rstrip": false,
59
  "single_word": false,
60
+ "special": false
61
  },
62
  "49113": {
63
  "content": "</tool_call>",
 
65
  "normalized": false,
66
  "rstrip": false,
67
  "single_word": false,
68
+ "special": false
69
  },
70
  "49114": {
71
  "content": "<tool_response>",
 
73
  "normalized": false,
74
  "rstrip": false,
75
  "single_word": false,
76
+ "special": false
77
  },
78
  "49115": {
79
  "content": "</tool_response>",
 
81
  "normalized": false,
82
  "rstrip": false,
83
  "single_word": false,
84
+ "special": false
85
  },
86
  "49116": {
87
  "content": "<think>",
 
89
  "normalized": false,
90
  "rstrip": false,
91
  "single_word": false,
92
+ "special": false
93
  },
94
  "49117": {
95
  "content": "</think>",
 
97
  "normalized": false,
98
  "rstrip": false,
99
  "single_word": false,
100
+ "special": false
101
  },
102
  "49118": {
103
  "content": "<answer>",
 
105
  "normalized": false,
106
  "rstrip": false,
107
  "single_word": false,
108
+ "special": false
109
  },
110
  "49119": {
111
  "content": "</answer>",
 
113
  "normalized": false,
114
  "rstrip": false,
115
  "single_word": false,
116
+ "special": false
117
  },
118
  "49120": {
119
  "content": "<context>",
 
121
  "normalized": false,
122
  "rstrip": false,
123
  "single_word": false,
124
+ "special": false
125
  },
126
  "49121": {
127
  "content": "</context>",
 
129
  "normalized": false,
130
  "rstrip": false,
131
  "single_word": false,
132
+ "special": false
133
  },
134
  "49122": {
135
  "content": "<|fim_prefix|>",
 
137
  "normalized": false,
138
  "rstrip": false,
139
  "single_word": false,
140
+ "special": false
141
  },
142
  "49123": {
143
  "content": "<|fim_suffix|>",
 
145
  "normalized": false,
146
  "rstrip": false,
147
  "single_word": false,
148
+ "special": false
149
  },
150
  "49124": {
151
  "content": "<|fim_middle|>",
 
153
  "normalized": false,
154
  "rstrip": false,
155
  "single_word": false,
156
+ "special": false
157
  },
158
  "49125": {
159
  "content": "<|repo_name|>",
 
161
  "normalized": false,
162
  "rstrip": false,
163
  "single_word": false,
164
+ "special": false
165
  },
166
  "49126": {
167
  "content": "<|image|>",
 
169
  "normalized": false,
170
  "rstrip": false,
171
  "single_word": false,
172
+ "special": false
173
  },
174
  "49127": {
175
  "content": "<|image_pad|>",
 
177
  "normalized": false,
178
  "rstrip": false,
179
  "single_word": false,
180
+ "special": false
181
  },
182
  "49128": {
183
  "content": "<|image_placeholder|>",
 
185
  "normalized": false,
186
  "rstrip": false,
187
  "single_word": false,
188
+ "special": false
189
  },
190
  "49129": {
191
  "content": " ",
 
193
  "normalized": false,
194
  "rstrip": false,
195
  "single_word": false,
196
+ "special": false
197
  },
198
  "49130": {
199
  "content": " ",
 
201
  "normalized": false,
202
  "rstrip": false,
203
  "single_word": false,
204
+ "special": false
205
  },
206
  "49131": {
207
  "content": " ",
 
209
  "normalized": false,
210
  "rstrip": false,
211
  "single_word": false,
212
+ "special": false
213
  },
214
  "49132": {
215
  "content": " ",
 
217
  "normalized": false,
218
  "rstrip": false,
219
  "single_word": false,
220
+ "special": false
221
  },
222
  "49133": {
223
  "content": " ",
 
225
  "normalized": false,
226
  "rstrip": false,
227
  "single_word": false,
228
+ "special": false
229
  },
230
  "49134": {
231
  "content": " ",
 
233
  "normalized": false,
234
  "rstrip": false,
235
  "single_word": false,
236
+ "special": false
237
  },
238
  "49135": {
239
  "content": " ",
 
241
  "normalized": false,
242
  "rstrip": false,
243
  "single_word": false,
244
+ "special": false
245
  },
246
  "49136": {
247
  "content": " ",
 
249
  "normalized": false,
250
  "rstrip": false,
251
  "single_word": false,
252
+ "special": false
253
  },
254
  "49137": {
255
  "content": " ",
 
257
  "normalized": false,
258
  "rstrip": false,
259
  "single_word": false,
260
+ "special": false
261
  },
262
  "49138": {
263
  "content": " ",
 
265
  "normalized": false,
266
  "rstrip": false,
267
  "single_word": false,
268
+ "special": false
269
  },
270
  "49139": {
271
  "content": " ",
 
273
  "normalized": false,
274
  "rstrip": false,
275
  "single_word": false,
276
+ "special": false
277
  },
278
  "49140": {
279
  "content": " ",
 
281
  "normalized": false,
282
  "rstrip": false,
283
  "single_word": false,
284
+ "special": false
285
  },
286
  "49141": {
287
  "content": " ",
 
289
  "normalized": false,
290
  "rstrip": false,
291
  "single_word": false,
292
+ "special": false
293
  },
294
  "49142": {
295
  "content": " ",
 
297
  "normalized": false,
298
  "rstrip": false,
299
  "single_word": false,
300
+ "special": false
301
  },
302
  "49143": {
303
  "content": " ",
 
305
  "normalized": false,
306
  "rstrip": false,
307
  "single_word": false,
308
+ "special": false
309
  },
310
  "49144": {
311
  "content": " ",
 
313
  "normalized": false,
314
  "rstrip": false,
315
  "single_word": false,
316
+ "special": false
317
  },
318
  "49145": {
319
  "content": " ",
 
321
  "normalized": false,
322
  "rstrip": false,
323
  "single_word": false,
324
+ "special": false
325
  },
326
  "49146": {
327
  "content": " ",
 
329
  "normalized": false,
330
  "rstrip": false,
331
  "single_word": false,
332
+ "special": false
333
  },
334
  "49147": {
335
  "content": " ",
 
337
  "normalized": false,
338
  "rstrip": false,
339
  "single_word": false,
340
+ "special": false
341
  },
342
  "49148": {
343
  "content": " ",
 
353
  "normalized": false,
354
  "rstrip": false,
355
  "single_word": false,
356
+ "special": false
357
  },
358
  "49150": {
359
  "content": " ",
 
361
  "normalized": false,
362
  "rstrip": false,
363
  "single_word": false,
364
+ "special": false
365
  },
366
  "49151": {
367
  "content": " ",
 
369
  "normalized": false,
370
  "rstrip": false,
371
  "single_word": false,
372
+ "special": false
373
  }
374
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
375
  "bos_token": "<|im_start|>",
376
  "bos_token_id": 1,
377
  "clean_up_tokenization_spaces": false,
 
379
  "eos_token_id": 2,
380
  "extra_special_tokens": {},
381
  "legacy": false,
382
+ "model_input_names": [
383
+ "input_ids",
384
+ "attention_mask"
385
+ ],
386
  "model_max_length": 4096,
387
  "pad_token": "<|pad|>",
388
  "pad_token_id": 49109,
 
390
  "sp_model_kwargs": {},
391
  "spaces_between_special_tokens": false,
392
  "tokenizer_class": "PreTrainedTokenizerFast",
 
 
 
 
393
  "truncation_side": "right",
394
  "unk_token": "<|unk|>",
395
  "unk_token_id": 0,