p04798526
/
LLaMA-Factory-310P3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1953 from ShaneTian/model-load-bf16 

Fix slow model initialization in bfloat16 dtype.
This commit is contained in:
hoshi-hiyouga 2023-12-22 17:29:54 +08:00 committed by GitHub
parent ba69378841 d032daa4bd
commit 315b8367cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/llmtuner/model/loader.py
View File

@ -67,10 +67,10 @@ def load_model_and_tokenizer(
model = AutoModelForCausalLM.from_pretrained( model = AutoModelForCausalLM.from_pretrained(
model_args.model_name_or_path, model_args.model_name_or_path,
config=config, config=config,
torch_dtype=model_args.compute_dtype,
low_cpu_mem_usage=(not is_deepspeed_zero3_enabled()), low_cpu_mem_usage=(not is_deepspeed_zero3_enabled()),
**config_kwargs **config_kwargs
) )
model = model.to(model_args.compute_dtype) if not getattr(model, "quantization_method", None) else model
patcher.patch_model(model) patcher.patch_model(model)
register_autoclass(config, model, tokenizer) register_autoclass(config, model, tokenizer)
if not is_deepspeed_zero3_enabled(): if not is_deepspeed_zero3_enabled():
@ -95,7 +95,6 @@ def load_model_and_tokenizer(
if not is_trainable: if not is_trainable:
model.requires_grad_(False) # fix all model params model.requires_grad_(False) # fix all model params
model = model.to(model_args.compute_dtype) if not getattr(model, "quantization_method", None) else model
model.eval() model.eval()
else: else:
model.train() model.train()