Joonas Pihlajamaa commited on
Commit
bf50d62
·
1 Parent(s): c95477e

Add enconding parameter to vocab.json opening to fix errors

Browse files
Files changed (1) hide show
  1. models/convert-pt-to-ggml.py +1 -1
models/convert-pt-to-ggml.py CHANGED
@@ -234,7 +234,7 @@ dir_tokenizer = tokenizer.name_or_path
234
  # output in the same directory as the model
235
  fname_out = dir_out + "/ggml-model.bin"
236
 
237
- with open(dir_tokenizer + "/vocab.json", "r") as f:
238
  tokens = json.load(f)
239
 
240
  # use 16-bit or 32-bit floats
 
234
  # output in the same directory as the model
235
  fname_out = dir_out + "/ggml-model.bin"
236
 
237
+ with open(dir_tokenizer + "/vocab.json", "r", encoding="utf8") as f:
238
  tokens = json.load(f)
239
 
240
  # use 16-bit or 32-bit floats