chore: add print

This commit is contained in:
wql 2024-09-26 14:00:14 +08:00
parent cc8d2e7ea0
commit 9905787eeb
1 changed files with 5 additions and 0 deletions

View File

@ -14,6 +14,7 @@
import os
import sys
import time
from typing import TYPE_CHECKING, Dict, Literal, Optional, Sequence, Union
import numpy as np
@ -212,6 +213,10 @@ def get_dataset(
if has_tokenized_data(data_args.tokenized_path):
logger.warning("Loading dataset from disk will ignore other data arguments.")
dataset_dict: "DatasetDict" = load_from_disk(data_args.tokenized_path)
print(data_args.tokenized_path)
print(dataset_dict)
time.sleep(100)
logger.info("Loaded tokenized dataset from {}.".format(data_args.tokenized_path))
dataset_module: Dict[str, "Dataset"] = {}