p04798526
/
LLaMA-Factory-Mirror
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix rm #420, fix template #426, fix #423

This commit is contained in:
hiyouga 2023-08-09 16:23:31 +08:00
parent 2d90685358
commit 39cd8b6989
3 changed files with 7 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/llmtuner/dsets/preprocess.py
View File

@ -103,13 +103,13 @@ def preprocess_dataset(
if len(source_ids) > data_args.max_source_length: if len(source_ids) > data_args.max_source_length:
source_ids = source_ids[:data_args.max_source_length] source_ids = source_ids[:data_args.max_source_length]
if len(accept_ids) > data_args.max_target_length - 1: # eos token if len(accept_ids) > data_args.max_target_length:
accept_ids = accept_ids[:data_args.max_target_length - 1] accept_ids = accept_ids[:data_args.max_target_length - 1]
if len(reject_ids) > data_args.max_target_length - 1: # eos token if len(reject_ids) > data_args.max_target_length:
reject_ids = reject_ids[:data_args.max_target_length - 1] reject_ids = reject_ids[:data_args.max_target_length - 1]
accept_ids = source_ids + accept_ids + [tokenizer.eos_token_id] accept_ids = source_ids + accept_ids
reject_ids = source_ids + reject_ids + [tokenizer.eos_token_id] reject_ids = source_ids + reject_ids
model_inputs["accept_ids"].append(accept_ids) model_inputs["accept_ids"].append(accept_ids)
model_inputs["reject_ids"].append(reject_ids) model_inputs["reject_ids"].append(reject_ids)

7
src/llmtuner/extras/template.py
View File

@ -388,12 +388,9 @@ register_template(
name="intern", name="intern",
prefix=[], prefix=[],
prompt=[ prompt=[
{"token": "<|User|>"}, "<|User|>:{{query}}",
":{{query}}",
{"token": "<eoh>"}, {"token": "<eoh>"},
"\n", "\n<|Bot|>:"
{"token": "<|Bot|>"},
":"
], ],
sep=[ sep=[
"\n" "\n"

2
src/llmtuner/webui/runner.py
View File

@ -156,7 +156,7 @@ class Runner:
else: else:
yield format_info(logger_handler.log, trainer_callback) yield format_info(logger_handler.log, trainer_callback)
if os.path.exists(os.path.join(output_dir), TRAINING_ARGS_NAME): if os.path.exists(os.path.join(output_dir, TRAINING_ARGS_NAME)):
finish_info = ALERTS["info_finished"][lang] finish_info = ALERTS["info_finished"][lang]
else: else:
finish_info = ALERTS["err_failed"][lang] finish_info = ALERTS["err_failed"][lang]