p04798526
/
LLaMA-Factory-310P3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove redundant code

This commit is contained in:
hiyouga 2023-08-05 00:27:27 +08:00
parent c183b3551d
commit 53d95725c5
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/llmtuner/extras/template.py
View File

@ -25,8 +25,8 @@ class Template:
r"""
Returns a single pair of token ids representing prompt and response respectively.
"""
prefix, history = self._format(query=query, resp=resp, history=history, prefix=prefix)
encoded_pairs = self._encode(tokenizer=tokenizer, prefix=prefix, history=history)
prefix, history = self._format(query, resp, history, prefix)
encoded_pairs = self._encode(tokenizer, prefix, history)
prompt_ids = []
for query_ids, resp_ids in encoded_pairs[:-1]:
prompt_ids = prompt_ids + query_ids + resp_ids
@ -44,8 +44,8 @@ class Template:
r"""
Returns multiple pairs of token ids representing prompts and responses respectively.
"""
prefix, history = self._format(query=query, resp=resp, history=history, prefix=prefix)
encoded_pairs = self._encode(tokenizer=tokenizer, prefix=prefix, history=history)
prefix, history = self._format(query, resp, history, prefix)
encoded_pairs = self._encode(tokenizer, prefix, history)
return encoded_pairs
def _format(