chore: add print
This commit is contained in:
parent
cc8d2e7ea0
commit
9905787eeb
|
@ -14,6 +14,7 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Dict, Literal, Optional, Sequence, Union
|
||||
|
||||
import numpy as np
|
||||
|
@ -212,6 +213,10 @@ def get_dataset(
|
|||
if has_tokenized_data(data_args.tokenized_path):
|
||||
logger.warning("Loading dataset from disk will ignore other data arguments.")
|
||||
dataset_dict: "DatasetDict" = load_from_disk(data_args.tokenized_path)
|
||||
print(data_args.tokenized_path)
|
||||
print(dataset_dict)
|
||||
time.sleep(100)
|
||||
|
||||
logger.info("Loaded tokenized dataset from {}.".format(data_args.tokenized_path))
|
||||
|
||||
dataset_module: Dict[str, "Dataset"] = {}
|
||||
|
|
Loading…
Reference in New Issue