p04798526
/
LLaMA-Factory-310P3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Slim Orca data parsing

This commit is contained in:
Mark Mueller 2024-02-08 17:56:18 +01:00
parent 7f792dfede
commit 6703d0546d
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/llmtuner/data/aligner.py
View File

@ -60,6 +60,7 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr"
if dataset_attr.system_tag and message[dataset_attr.role_tag] == dataset_attr.system_tag: if dataset_attr.system_tag and message[dataset_attr.role_tag] == dataset_attr.system_tag:
outputs["system"].append(message[dataset_attr.content_tag]) outputs["system"].append(message[dataset_attr.content_tag])
n_sys = 1 n_sys = 1
continue
if (turn_idx - n_sys) % 2 == 0: if (turn_idx - n_sys) % 2 == 0:
accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag] accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag]