Spaces:
Running
Running
Joonas Pihlajamaa
commited on
Commit
·
bf50d62
1
Parent(s):
c95477e
Add enconding parameter to vocab.json opening to fix errors
Browse files
models/convert-pt-to-ggml.py
CHANGED
|
@@ -234,7 +234,7 @@ dir_tokenizer = tokenizer.name_or_path
|
|
| 234 |
# output in the same directory as the model
|
| 235 |
fname_out = dir_out + "/ggml-model.bin"
|
| 236 |
|
| 237 |
-
with open(dir_tokenizer + "/vocab.json", "r") as f:
|
| 238 |
tokens = json.load(f)
|
| 239 |
|
| 240 |
# use 16-bit or 32-bit floats
|
|
|
|
| 234 |
# output in the same directory as the model
|
| 235 |
fname_out = dir_out + "/ggml-model.bin"
|
| 236 |
|
| 237 |
+
with open(dir_tokenizer + "/vocab.json", "r", encoding="utf8") as f:
|
| 238 |
tokens = json.load(f)
|
| 239 |
|
| 240 |
# use 16-bit or 32-bit floats
|