p04798526
/
LLaMA-Factory-310P3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix #608

This commit is contained in:
hiyouga 2023-08-21 17:49:36 +08:00
parent 0a3f698425
commit 02d69b6fde
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/llmtuner/extras/template.py
View File

@ -224,7 +224,10 @@ def get_template_and_fix_tokenizer(
tokenizer.pad_token = tokenizer.eos_token
logger.info("Add pad token: {}".format(tokenizer.pad_token))
tokenizer.add_special_tokens(dict(additional_special_tokens=additional_special_tokens))
tokenizer.add_special_tokens(
dict(additional_special_tokens=additional_special_tokens),
replace_additional_special_tokens=False
)
return template
@ -472,7 +475,7 @@ register_template(
r"""
Supports: https://huggingface.co/baichuan-inc/Baichuan-13B-Chat
Used for training.
Used for training and inference of the fine-tuned models.
"""
register_template(
name="baichuan",