From c183b3551d8d190965d2403e27fa2cedf8ac7bff Mon Sep 17 00:00:00 2001 From: hiyouga Date: Sat, 5 Aug 2023 00:25:00 +0800 Subject: [PATCH] fix template --- src/llmtuner/extras/template.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/llmtuner/extras/template.py b/src/llmtuner/extras/template.py index 0d6eb57c..57d304ab 100644 --- a/src/llmtuner/extras/template.py +++ b/src/llmtuner/extras/template.py @@ -101,9 +101,7 @@ class Template: token_ids = [] for elem in context: if isinstance(elem, str): - subelems = elem.split("{{query}}") - if len(subelems) > 1: - elem = subelems[0] + query + subelems[1] + elem = elem.replace("{{query}}", query, 1) token_ids = token_ids + tokenizer.encode(elem, add_special_tokens=False) elif isinstance(elem, dict): token_ids = token_ids + [tokenizer.convert_tokens_to_ids(elem.get("token"))]