p83651209
/
CPM-9G-8B
forked from jiuyuan/CPM-9G-8B
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: p83651209:FM_9G
Branches Tags
p83651209:FM_9G
p83651209:new
p71924506:FM_9G
p71924506:new
jiuyuan:master
jiuyuan:FM_9G
..
compare: p71924506:FM_9G
Branches Tags
p71924506:FM_9G
p71924506:new
p83651209:FM_9G
p83651209:new
jiuyuan:master
jiuyuan:FM_9G

No commits in common. "FM_9G" and "FM_9G" have entirely different histories.
FM_9G ... FM_9G

3 changed files with 289 additions and 314 deletions
Show Stats Expand all files Collapse all files

41
README.md
View File

@ -1,27 +1,24 @@
夸克网盘 docker链接：https://pan.quark.cn/s/4cda395f13e8 方案：
(没有会员请联系我下载) 全参数微调，使用不同数据集训练多个模型和推理时增强进行融合。
1.使用llama-factory对九格模型进行全参数微调。数据集见dataset 训练代码：
LLaMA-Factory.zip 解压后使用，可参照https://github.com/hiyouga/LLaMA-Factory配置环境，或将代码映射到docker中使用。
训练：train.sh。将数据集放到LLaMA-Factory/data文件夹下，将train.sh放到LLaMA-Factory下使用。
推理： python inference.py(需在inference.py中修改好模型路径。) test_case.json是从题目中提取出来的测试用例。
2.训练和推理都已验证无误，在A100*8卡机器上。 百度网盘需要收费，使用阿里云盘
docker 启动：sudo docker run -it --runtime=nvidia --gpus all --shm-size=256g wjf:train model_wight:通过百度网盘分享的文件：
推理：python inference.py 链接：https://pan.baidu.com/s/1paYNO7d5OYESuyw3BVo7Ew
训练： 提取码：6666
cd training https://www.alipan.com/s/FTPWUSBuz7s
sh training.sh
docker:
链接：https://pan.baidu.com/s/1paYNO7d5OYESuyw3BVo7Ew
提取码：6666
https://www.alipan.com/s/FTPWUSBuz7s
3.推理使用多checkpoint、多次推理融合。 train_data:
链接：https://pan.baidu.com/s/1paYNO7d5OYESuyw3BVo7Ew
4.所有资料都已打包进docker，只需要docker即可。 提取码：6666
https://www.alipan.com/s/FTPWUSBuz7s
5.启动训练时将覆盖提交的checkpoint。
6.docker卡在数据处理可能是机器的问题，尝试docker中输入：
export NCCL_DEBUG=INFO
export NCCL_SHM_DISABLE=1
export NCCL_P2P_DISABLE=1
由于需要保存多个checkpoint，请务必保证磁盘空间足够，大于500G。
7.提交不易，请有问题是及时联系我（电话：13121813131）

548
inference.py
View File

@ -1,278 +1,270 @@
import json, torch, re, sys, subprocess import json, torch, re, sys, subprocess
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel, StoppingCriteria from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel, StoppingCriteria
device = "cuda" # the device to load the model onto device = "cuda" # the device to load the model onto
from tqdm import tqdm from tqdm import tqdm
def exec_code(test): def exec_code(test):
with open("test_case.json", "r") as f: with open("test_case.json", "r") as f:
test_cases = json.load(f) test_cases = json.load(f)
right_num = 0 right_num = 0
all_num = 0 all_num = 0
package = "import os, sys, math, re, json, random\n" package = "import os, sys, math, re, json, random\n"
for item, test_case in zip(test, test_cases): for item, test_case in zip(test, test_cases):
if "```python\n" in item["raw_outputs"]: if "```python\n" in item["raw_outputs"]:
matches = re.findall('```python(.*?)```', item["raw_outputs"], re.DOTALL) matches = re.findall('```python(.*?)```', item["raw_outputs"], re.DOTALL)
if len(matches) == 1: if len(matches) == 1:
item["raw_outputs"] = matches[0] item["raw_outputs"] = matches[0]
else: else:
matches = re.findall('```python(.*?)assert', item["raw_outputs"], re.DOTALL) matches = re.findall('```python(.*?)assert', item["raw_outputs"], re.DOTALL)
if len(matches) == 1: if len(matches) == 1:
item["raw_outputs"] = matches[0] item["raw_outputs"] = matches[0]
else: else:
item["raw_outputs"] = item["raw_outputs"][item["raw_outputs"].index("python\n") + len("python\n"):] item["raw_outputs"] = item["raw_outputs"][item["raw_outputs"].index("python\n") + len("python\n"):]
print(item) print(item)
#break #break
code = item["raw_outputs"].replace("<|im_end|>", "").replace("</s>", "").replace("```", "").strip().rstrip("\n") code = item["raw_outputs"].replace("<|im_end|>", "").replace("</s>", "").replace("```", "").strip().rstrip("\n")
raw_code = code raw_code = code
codes = raw_code.split("\n") codes = raw_code.split("\n")
last_line = 0 last_line = 0
for index, line in enumerate(codes): for index, line in enumerate(codes):
if " return" in line: if " return" in line:
last_line = index last_line = index
code = "\n".join(codes[:last_line+1]) code = "\n".join(codes[:last_line+1])
''' '''
if raw_code != code: if raw_code != code:
print("\n--------------------------------------------------------\n", [raw_code], "\n--------------------------------------------------------\n") print("\n--------------------------------------------------------\n", [raw_code], "\n--------------------------------------------------------\n")
print("clean:\n", [code], "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n\n") print("clean:\n", [code], "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n\n")
''' '''
with open('code_.py', 'w') as fout: with open('code_.py', 'w') as fout:
fout.write(package + code + "\n" + "\n".join(test_case["test_case"])) fout.write(package + code + "\n" + "\n".join(test_case["test_case"]))
batcmd = 'timeout 3 ' + sys.executable + ' code_.py' batcmd = 'timeout 3 ' + sys.executable + ' code_.py'
try: try:
shell_output = subprocess.check_output(batcmd, shell=True).decode('utf8') shell_output = subprocess.check_output(batcmd, shell=True).decode('utf8')
right_num += 1 right_num += 1
item["result"] = "True" item["result"] = "True"
except Exception as e: except Exception as e:
print("++++++++++++++++++++++++++++++++++++++++++++++++++++\n", raw_code, "\n-----------------------------------------\n\n\n", package + code + "\n--------------------------\n" + "\n".join(test_case["test_case"])) print("++++++++++++++++++++++++++++++++++++++++++++++++++++\n", raw_code, "\n-----------------------------------------\n\n\n", package + code + "\n--------------------------\n" + "\n".join(test_case["test_case"]))
print("--------------------------------------------------------\n\n\nitem:", item) print("--------------------------------------------------------\n\n\nitem:", item)
print("e: ", e, "\n================================================\n")#, e, ) print("e: ", e, "\n================================================\n")#, e, )
item["result"] = "False" item["result"] = "False"
all_num += 1 all_num += 1
item["raw_outputs"] = [code] item["raw_outputs"] = [code]
print(len(test), right_num, all_num, right_num / all_num) print(len(test), right_num, all_num, right_num / all_num)
with open(f'wjf_{model_path.replace("/", "-")}{right_num / all_num}.json', "w") as f: with open(f'wjf_{model_path.replace("/", "-")}{right_num / all_num}.json', "w") as f:
json.dump(test, f, indent=4) json.dump(test, f, indent=4)
return test, right_num / all_num return test, right_num / all_num
def get_result(model, tokenizer): def get_result(model, tokenizer):
test = [] test = []
with open("/mnt/disk2/home/wujianfeng/com/code/code_round4.jsonl", "r") as f: with open("/mnt/disk2/home/wujianfeng/com/code/code_round4.jsonl", "r") as f:
#test = json.load(f) #test = json.load(f)
for line in f: for line in f:
test.append(json.loads(line)) test.append(json.loads(line))
all_score = 0 all_score = 0
all_num = 0 all_num = 0
test_num = 1000 test_num = 1000
from tqdm import tqdm from tqdm import tqdm
for example in tqdm(test[:]): for example in tqdm(test[:]):
#print(example["question"]) #print(example["question"])
example["question"] = example["question"].replace("'''", '"""') example["question"] = example["question"].replace("'''", '"""')
ai_prefix = "" ai_prefix = ""
if example["question"].split(" ")[0] == "Write": if example["question"].split(" ")[0] == "Write":
question = example["question"][:example["question"].index("\n")].strip().rstrip() question = example["question"][:example["question"].index("\n")].strip().rstrip()
test_case = example["question"][example["question"].index("\n"):].split("\n") test_case = example["question"][example["question"].index("\n"):].split("\n")
print("test_case: ", test_case) print("test_case: ", test_case)
function_name = test_case[1].split(" ")[1].split("(")[0] function_name = test_case[1].split(" ")[1].split("(")[0]
ai_prefix = "def " + function_name ai_prefix = "def " + function_name
messages = [ messages = [
{"role": "user", "content": question + "\n\n" + ("\n".join(test_case))} {"role": "user", "content": question + "\n\n" + ("\n".join(test_case))}
] ]
text = tokenizer.apply_chat_template( text = tokenizer.apply_chat_template(
messages, messages,
tokenize=False, tokenize=False,
add_generation_prompt=True add_generation_prompt=True
) )
text += ai_prefix text += ai_prefix
example["test_case"] = test_case example["test_case"] = test_case
else: else:
tmp = re.findall(r'"""(.*?)"""', example["question"], flags=re.DOTALL)[0].split("\n") tmp = re.findall(r'"""(.*?)"""', example["question"], flags=re.DOTALL)[0].split("\n")
question = "" question = ""
for line in tmp: for line in tmp:
line = line.strip().rstrip() line = line.strip().rstrip()
if len(line) == 0: if len(line) == 0:
continue continue
#if "xample" in line and len(line) < 20: #if "xample" in line and len(line) < 20:
# break # break
question += line + " " question += line + " "
code = re.sub(r'"""(.*?)"""', '', example["question"], flags=re.DOTALL).strip().rstrip() code = re.sub(r'"""(.*?)"""', '', example["question"], flags=re.DOTALL).strip().rstrip()
ai_prefix = code ai_prefix = code
messages = [ messages = [
{"role": "user", "content": question} {"role": "user", "content": question}
] ]
text = tokenizer.apply_chat_template( text = tokenizer.apply_chat_template(
messages, messages,
tokenize=False, tokenize=False,
add_generation_prompt=True add_generation_prompt=True
) )
text += ai_prefix text += ai_prefix
example["prompt"] = text example["prompt"] = text
print("text: " , [text]) print("text: " , [text])
input_ids = tokenizer([text], return_tensors="pt").to(device).input_ids input_ids = tokenizer([text], return_tensors="pt").to(device).input_ids
output = model.generate(input_ids, output = model.generate(input_ids,
#top_p=1.0, #top_p=1.0,
max_new_tokens=600, max_new_tokens=600,
#repetition_penalty=1.1 + t*0.01, #repetition_penalty=1.1 + t*0.01,
temperature=0.1, temperature=0.1,
#no_repeat_ngram_size = 5, #no_repeat_ngram_size = 5,
).squeeze() ).squeeze()
output_str = tokenizer.decode(output[input_ids.shape[1]:]) output_str = tokenizer.decode(output[input_ids.shape[1]:])
output_str = ai_prefix + output_str output_str = ai_prefix + output_str
print("output_str:\n", output_str, "\n-----------------------------------------------------------------") print("output_str:\n", output_str, "\n-----------------------------------------------------------------")
example["raw_outputs"] = output_str#re.findall(r'```python(.*?)```', output_str) example["raw_outputs"] = output_str#re.findall(r'```python(.*?)```', output_str)
return test return test
def get_result_1(model, tokenizer): def get_result_1(model, tokenizer):
test = [] test = []
with open("/mnt/disk2/home/wujianfeng/com/code/code_round4.jsonl", "r") as f: with open("/mnt/disk2/home/wujianfeng/com/code/code_round4.jsonl", "r") as f:
#test = json.load(f) #test = json.load(f)
for line in f: for line in f:
test.append(json.loads(line)) test.append(json.loads(line))
all_score = 0 all_score = 0
all_num = 0 all_num = 0
test_num = 1000 test_num = 1000
from tqdm import tqdm from tqdm import tqdm
for example in tqdm(test[:]): for example in tqdm(test[:]):
#print(example["question"]) #print(example["question"])
messages = [ messages = [
{"role": "user", "content": example["question"]} {"role": "user", "content": example["question"]}
] ]
text = tokenizer.apply_chat_template( text = tokenizer.apply_chat_template(
messages, messages,
tokenize=False, tokenize=False,
add_generation_prompt=True add_generation_prompt=True
) )
example["prompt"] = text example["prompt"] = text
print("text: " , [text]) print("text: " , [text])
input_ids = tokenizer([text], return_tensors="pt").to(device).input_ids input_ids = tokenizer([text], return_tensors="pt").to(device).input_ids
output = model.generate(input_ids, output = model.generate(input_ids,
#top_p=1.0, #top_p=1.0,
max_new_tokens=600, max_new_tokens=600,
#repetition_penalty=1.1 + t*0.01, #repetition_penalty=1.1 + t*0.01,
temperature=0.1, temperature=0.1,
#no_repeat_ngram_size = 5, #no_repeat_ngram_size = 5,
).squeeze() ).squeeze()
output_str = tokenizer.decode(output[input_ids.shape[1]:]) output_str = tokenizer.decode(output[input_ids.shape[1]:])
print("output_str:\n", output_str, "\n-----------------------------------------------------------------") print("output_str:\n", output_str, "\n-----------------------------------------------------------------")
example["raw_outputs"] = output_str#re.findall(r'```python(.*?)```', output_str) example["raw_outputs"] = output_str#re.findall(r'```python(.*?)```', output_str)
return test return test
answers = {} answers = {}
for model_path in [
for model_path in [ "/mnt/disk2/home/wujianfeng/LLaMA-Factory/all_new_1/checkpoint-600",
"/mnt/disk2/home/wujianfeng/LLaMA-Factory/all/TACO/", "/mnt/disk2/home/wujianfeng/LLaMA-Factory/all_new/checkpoint-600/",
"/mnt/disk2/home/wujianfeng/LLaMA-Factory/all_new_2/CodeNet4Repair/",
"/mnt/disk2/home/wujianfeng/LLaMA-Factory/all_new_1/CodeExercise-Python-27k/", ]:
]: print("model_path: ", model_path)
print("model_path: ", model_path) model = AutoModelForCausalLM.from_pretrained(
model = AutoModelForCausalLM.from_pretrained( model_path,
model_path, torch_dtype="auto",
torch_dtype="auto", device_map=device,
device_map=device, trust_remote_code=True,
trust_remote_code=True, )
) tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
test = get_result(model, tokenizer)
test = get_result(model, tokenizer) test, score = exec_code(test)
test, score = exec_code(test) answers[score] = test
answers[score] = test
test = get_result_1(model, tokenizer)
test = get_result_1(model, tokenizer) test, score = exec_code(test)
test, score = exec_code(test) answers[score] = test
answers[score] = test
'''
import os answers = list(dict(sorted(answers.items())).values())
for path in os.listdir("./"): print("answers: ", answers)
if "home-wujianfeng" in path: right = 0
with open(path, "r") as f: jiuge_right = 0
test = json.load(f) merge = []
answers[float(path.split(".")[-2].split("-")[-1])] = test for i in range(len(answers)):
''' #for i in range(2):
flag = 0
answers = list(dict(sorted(answers.items())).values()) for answer in answers:
print("answers: ", answers) if answer[i]["result"] == "True":
right = 0 right += 1
jiuge_right = 0 jiuge_right += 1
merge = [] flag = 1
for i in range(len(answers[0])): merge.append(answer[i])
#for i in range(2): break
flag = 0
for answer in answers: if flag == 0:
if answer[i]["result"] == "True": merge.append(answers[0][i])
right += 1
jiuge_right += 1
flag = 1
merge.append(answer[i]) print(right / len(answers), jiuge_right / len(answers))
break with open("wjf_jiuge.jsonl", "w") as f:
for item in merge:
if flag == 0: item.pop("result")
merge.append(answers[0][i]) f.write(json.dumps(item, ensure_ascii=False) + '\n')
print(right / len(answers[0]), jiuge_right / len(answers[0]))
with open("wjf_jiuge.jsonl", "w") as f:
for item in merge:
item.pop("result")
f.write(json.dumps(item, ensure_ascii=False) + '\n')

14
model_final_url.txt
View File

@ -1,14 +0,0 @@
model_wight:通过百度网盘分享的文件：
链接：https://pan.baidu.com/s/1paYNO7d5OYESuyw3BVo7Ew
提取码：6666
#https://www.alipan.com/s/FTPWUSBuz7s
docker:
链接：https://pan.baidu.com/s/1paYNO7d5OYESuyw3BVo7Ew
提取码：6666
#https://www.alipan.com/s/FTPWUSBuz7s
train_data:
链接：https://pan.baidu.com/s/1paYNO7d5OYESuyw3BVo7Ew
提取码：6666
#https://www.alipan.com/s/FTPWUSBuz7s