From e21ae0135610bad8116cadbe4b184aac8e279d7c Mon Sep 17 00:00:00 2001 From: hiyouga Date: Sun, 6 Aug 2023 13:31:17 +0800 Subject: [PATCH] fix qwen eos token --- src/llmtuner/tuner/core/loader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/llmtuner/tuner/core/loader.py b/src/llmtuner/tuner/core/loader.py index 59afbf38..5610deef 100644 --- a/src/llmtuner/tuner/core/loader.py +++ b/src/llmtuner/tuner/core/loader.py @@ -68,7 +68,9 @@ def load_model_and_tokenizer( padding_side=model_args.padding_side, **config_kwargs ) - if tokenizer.pad_token_id is None or tokenizer.pad_token_id == 64000: # 64000 for baichuan model (older version) + if tokenizer.eos_token_id is None: # fix qwen tokenizer + tokenizer.eos_token = "<|endoftext|>" + if tokenizer.pad_token_id is None: # add pad token tokenizer.pad_token = tokenizer.eos_token config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)