update cache dir

2023-12-12 13:08:18 +08:00 · 2023-12-12 13:08:18 +08:00 · edc82b923a
parent 09533e95ed
commit edc82b923a
1 changed files with 3 additions and 1 deletions
--- a/src/llmtuner/data/loader.py
+++ b/src/llmtuner/data/loader.py
@ -55,12 +55,14 @@ def get_dataset(

        if int(os.environ.get('USE_MODELSCOPE_HUB', '0')) and dataset_attr.load_from == "ms_hub":
            from modelscope import MsDataset
+            cache_dir = model_args.cache_dir
+            cache_dir = str(cache_dir) if cache_dir is not None else None
            dataset = MsDataset.load(
                dataset_name=data_path,
                subset_name=data_name,
                split=data_args.split,
                data_files=data_files,
-                cache_dir=model_args.cache_dir,
+                cache_dir=cache_dir,
                token=model_args.ms_hub_token,
                streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
            ).to_hf_dataset()