File size: 571 Bytes
e875200 |
1 2 3 4 5 6 7 8 9 10 |
from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, AutoTokenizer
from transformers.tokenization_utils_fast import PreTrainedTokenizerFast
from configuration_crystalcoder import CrystalCoderConfig
from modeling_crystalcoder import CrystalCoderModel, CrystalCoderLMHeadModel
AutoConfig.register("crystalcoder", CrystalCoderConfig)
AutoModel.register(CrystalCoderConfig, CrystalCoderModel)
AutoModelForCausalLM.register(CrystalCoderConfig, CrystalCoderLMHeadModel)
AutoTokenizer.register(CrystalCoderConfig, fast_tokenizer_class=PreTrainedTokenizerFast)
|