p04798526
/
LLaMA-Factory-Mirror
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update cache dir

This commit is contained in:
xingjun.wang 2023-12-12 13:08:18 +08:00
parent 09533e95ed
commit edc82b923a
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/data/loader.py
View File

@ -55,12 +55,14 @@ def get_dataset(
if int(os.environ.get('USE_MODELSCOPE_HUB', '0')) and dataset_attr.load_from == "ms_hub": if int(os.environ.get('USE_MODELSCOPE_HUB', '0')) and dataset_attr.load_from == "ms_hub":
from modelscope import MsDataset from modelscope import MsDataset
cache_dir = model_args.cache_dir
cache_dir = str(cache_dir) if cache_dir is not None else None
dataset = MsDataset.load( dataset = MsDataset.load(
dataset_name=data_path, dataset_name=data_path,
subset_name=data_name, subset_name=data_name,
split=data_args.split, split=data_args.split,
data_files=data_files, data_files=data_files,
cache_dir=model_args.cache_dir, cache_dir=cache_dir,
token=model_args.ms_hub_token, token=model_args.ms_hub_token,
streaming=(data_args.streaming and (dataset_attr.load_from != "file")), streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
).to_hf_dataset() ).to_hf_dataset()