From 4955dc9eed33a904e4e2b9d5985b3fda87c3674a Mon Sep 17 00:00:00 2001
From: zxbsmk <3028208898@qq.com>
Date: Wed, 12 Jul 2023 17:29:47 +0800
Subject: [PATCH] Support for WebNovel dataset

---
 README.md              | 1 +
 data/dataset_info.json | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 4495e697..141587fd 100644
--- a/README.md
+++ b/README.md
@@ -73,6 +73,7 @@
   - [UltraChat](https://github.com/thunlp/UltraChat)
   - [Open Assistant](https://huggingface.co/datasets/OpenAssistant/oasst1)
   - [Open Assistant (Chinese)](https://huggingface.co/datasets/OpenAssistant/oasst1)
+  - [WebNovel (Chinese)](https://huggingface.co/datasets/zxbsmk/webnovel_cn)
 - For reward model training:
   - [HH-RLHF](https://huggingface.co/datasets/Anthropic/hh-rlhf)
   - [Open Assistant](https://huggingface.co/datasets/OpenAssistant/oasst1)
diff --git a/data/dataset_info.json b/data/dataset_info.json
index 5ad42e64..a4323e3b 100644
--- a/data/dataset_info.json
+++ b/data/dataset_info.json
@@ -149,5 +149,8 @@
       "response": "",
       "history": ""
     }
+  },
+  "novel_tokens512_50k": {
+    "hf_hub_url": "zxbsmk/webnovel_cn"
   }
 }