Fix tokenizer reloading
#42
by
kylesayrs
- opened
- tokenization_chatglm.py +2 -0
tokenization_chatglm.py
CHANGED
|
@@ -54,6 +54,8 @@ class ChatGLM4Tokenizer(PreTrainedTokenizer):
|
|
| 54 |
super().__init__(
|
| 55 |
padding_side=padding_side,
|
| 56 |
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
|
|
|
|
|
|
| 57 |
**kwargs
|
| 58 |
)
|
| 59 |
|
|
|
|
| 54 |
super().__init__(
|
| 55 |
padding_side=padding_side,
|
| 56 |
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 57 |
+
encode_special_tokens=encode_special_tokens,
|
| 58 |
+
image_size=image_size,
|
| 59 |
**kwargs
|
| 60 |
)
|
| 61 |
|