520 lines
13 KiB
Python
520 lines
13 KiB
Python
import pandas as pd
|
||
import numpy as np
|
||
import tensorflow as tf
|
||
import keras
|
||
from sklearn.preprocessing import MinMaxScaler
|
||
import re
|
||
import copy
|
||
from random import shuffle
|
||
from random import uniform
|
||
import math
|
||
|
||
#辅助函数
|
||
#myinput必须是string类型
|
||
def InputToSenList(senten,model):
|
||
mark=' mark! '
|
||
#使用正则表达式确定是否要切分
|
||
stripSpecialChars=re.compile("[^A-Za-z0-9 ]+")
|
||
#把大写字母改成小写字母
|
||
senten=senten.lower().replace('<br />','')
|
||
#print(senten)
|
||
#把所有的标点符号更换为mark
|
||
subSenList=[]
|
||
|
||
if model=='clause':
|
||
myinput=re.sub(stripSpecialChars,mark,senten)
|
||
#wordVec保存的是token，即单词
|
||
wordVec=myinput.split()
|
||
|
||
#markLoc保存mark！的位置，这就是标点符号的位置，作为切分子句的依据
|
||
markLoc=[]
|
||
markLoc.append(0)
|
||
|
||
shiftNum=0
|
||
for i in range(len(wordVec)):
|
||
if wordVec[i-shiftNum]=='mark!':
|
||
markLoc.append(i-shiftNum)
|
||
wordVec.pop(i-shiftNum)
|
||
shiftNum+=1
|
||
|
||
#按照标点符号划分子句，把每个子句放入subSenList
|
||
for i in range(len(markLoc)-1):
|
||
subSenList.append(" ".join(wordVec[markLoc[i]:markLoc[i+1]]))
|
||
else:
|
||
myinput=re.sub(stripSpecialChars,' ',senten)
|
||
#wordVec保存的是token，即单词
|
||
subSenList=myinput.split()
|
||
|
||
return subSenList
|
||
|
||
|
||
|
||
|
||
|
||
|
||
#把list转化为词嵌入向量的comment
|
||
def ListToVecComment(tempSenList):
|
||
global wordsList
|
||
|
||
comment=np.zeros([batchSize,maxSeqLength])
|
||
#comment保存的评论，每个字都是用字典中的数字组成的
|
||
|
||
|
||
fullSent=' '.join(tempSenList)
|
||
|
||
counter=0
|
||
|
||
for word in fullSent.split():
|
||
try:
|
||
comment[0][counter]=wordsList.index(word)
|
||
except Exception:
|
||
comment[0][counter]=399999
|
||
counter+=1
|
||
if counter==250:
|
||
break
|
||
|
||
return comment
|
||
|
||
|
||
|
||
def CountWordNum(strs):
|
||
count=1
|
||
for word in strs:
|
||
if word==' ':
|
||
count+=1
|
||
|
||
return count
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def GetSenList(myinput,model='clause'):
|
||
|
||
senList=[]
|
||
#只预测一天，只要最后十五天的数据
|
||
tempList=myinput.split()[-15:]
|
||
|
||
if model=='word':
|
||
senList=tempList
|
||
else:
|
||
senten = ''
|
||
count = 0
|
||
for number in tempList:
|
||
senten += str(number)+' '
|
||
count += 1
|
||
if(count>=3):
|
||
senList.append(senten)
|
||
senten=''
|
||
count=0
|
||
|
||
if senten:
|
||
senList.append(senten)
|
||
|
||
|
||
return senList
|
||
|
||
|
||
|
||
|
||
|
||
|
||
maxSeqLength=250
|
||
|
||
batchSize = 24
|
||
lstmUnits = 64
|
||
numClasses = 2
|
||
numDimensions = 50 #Dimensions for each word vector
|
||
iterations = 1000 #100000
|
||
learnRate=0
|
||
|
||
|
||
#辅助函数
|
||
from random import randint
|
||
|
||
|
||
|
||
|
||
class GetDifferCovid():
|
||
def __init__(self,myinput,RNNModel):
|
||
#拆分好的句子的list
|
||
self.senList=InputToSenList(myinput,'clause')
|
||
#拆分好的单词的list
|
||
self.wordList=InputToSenList(myinput,'word')
|
||
#根据选择的句子/单词 model 确定的list
|
||
|
||
#计算到的list长度
|
||
self.sentenSize=len(self.senList)
|
||
|
||
|
||
#定义好的RNN
|
||
# self.PreRNN=PredictRNN()
|
||
self.PreRNN=RNNModel
|
||
#原序列的预测值
|
||
#使用者的模型中必须有一个predict的方法
|
||
self.oriRes=self.PreRNN.Predict(' '.join(self.senList))
|
||
self.iterations=100
|
||
#设置循环的次数，建议是总的子句数量*整体循环次数
|
||
|
||
|
||
#用来寻找偏差最大的序列
|
||
def GetDiffOrder(self):
|
||
|
||
|
||
|
||
indexOrder1= np.arange(start=0,stop=self.sentenSize,dtype=np.int)
|
||
|
||
|
||
indexOrder1=indexOrder1.reshape(1,-1)
|
||
indexOrder1=np.repeat(indexOrder1,batchSize,axis=0)
|
||
#24*子句长度
|
||
indexOrder2=copy.deepcopy(indexOrder1)
|
||
|
||
|
||
|
||
fitness=np.zeros(batchSize*2)
|
||
|
||
allRes=None
|
||
allIndexOrder=None
|
||
|
||
iterations=15
|
||
for i in range(iterations):
|
||
indexOrder1 = self.Variate(indexOrder1,i)
|
||
indexOrder2 = self.Variate(indexOrder2,i)
|
||
|
||
# if i!=(iterations-1):
|
||
|
||
indexOrder1=self.OX(indexOrder1)
|
||
indexOrder2=self.OX(indexOrder2)
|
||
|
||
|
||
comment1=self.IndexToInput(indexOrder1)
|
||
|
||
res1=np.arange(start=0,stop=batchSize,dtype=np.float32)
|
||
for i in range(len(comment1)):
|
||
res1[i]=self.PreRNN.Predict(comment1[i])
|
||
|
||
|
||
|
||
comment2=self.IndexToInput(indexOrder2)
|
||
|
||
res2=np.arange(start=0,stop=batchSize,dtype=np.float32)
|
||
for i in range(len(comment2)):
|
||
res1[i]=self.PreRNN.Predict(comment2[i])
|
||
|
||
allRes = np.concatenate((res1,res2))
|
||
allIndexOrder = np.concatenate((indexOrder1,indexOrder2))
|
||
|
||
#计算适应度函数
|
||
#偏差值*20-逆序数
|
||
for j in range(batchSize*2):
|
||
fitness[j] = (allRes[j]-self.oriRes) #(res[j][0]-oriRes[0])*30
|
||
#fitness[j] -= InverNum(indexOrder[j])
|
||
|
||
|
||
|
||
totalRan=0
|
||
|
||
|
||
|
||
for re in allRes:
|
||
totalRan+=(abs(re-self.oriRes))
|
||
|
||
for j in range(batchSize):
|
||
ranNum=uniform(0,totalRan)
|
||
ranCount=0
|
||
for k in range(len(allRes)):
|
||
ranCount+=(abs(allRes[k]-self.oriRes))
|
||
if(ranCount>=ranNum):
|
||
|
||
indexOrder1[j]=allIndexOrder[k]
|
||
break
|
||
|
||
|
||
|
||
for j in range(batchSize*2):
|
||
fitness[j] = (self.oriRes-allRes[j])
|
||
#(res[j][0]-oriRes[0])*30
|
||
#fitness[j] -= InverNum(indexOrder[j])
|
||
|
||
|
||
|
||
totalRan=0
|
||
|
||
|
||
|
||
for re in allRes:
|
||
totalRan+=(abs(re-self.oriRes))
|
||
|
||
for j in range(batchSize):
|
||
ranNum=uniform(0,totalRan)
|
||
ranCount=0
|
||
for k in range(len(allRes)):
|
||
ranCount+=(abs(allRes[k]-self.oriRes))
|
||
if(ranCount>=ranNum):
|
||
indexOrder2[j]=allIndexOrder[k]
|
||
break
|
||
|
||
|
||
# else:
|
||
# indexOrder=fatherIndexOrder
|
||
|
||
|
||
|
||
|
||
comment1=self.IndexToInput(indexOrder1)
|
||
|
||
res1=np.arange(start=0,stop=batchSize,dtype=np.float32)
|
||
for i in range(len(comment1)):
|
||
res1[i]=self.PreRNN.Predict(comment1[i])
|
||
|
||
comment2=self.IndexToInput(indexOrder2)
|
||
|
||
|
||
res2=np.arange(start=0,stop=batchSize,dtype=np.float32)
|
||
for i in range(len(comment2)):
|
||
res1[i]=self.PreRNN.Predict(comment2[i])
|
||
|
||
|
||
allRes = np.concatenate((res1,res2))
|
||
allIndexOrder = np.concatenate((indexOrder1,indexOrder2))
|
||
|
||
|
||
for j in range(1, batchSize*2):
|
||
for k in range(0, batchSize*2 - j ):
|
||
if allRes[k]> allRes[k+1]:
|
||
allRes[k],allRes[k+1] = allRes[k+1],allRes[k]
|
||
allIndexOrder[[k,k+1], :] = allIndexOrder[[k+1,k], :]
|
||
|
||
|
||
# allRes[k], allRes[k+1] = allRes[k+1], allRes[k]
|
||
# allIndexOrder[k], allIndexOrder[k+1] = allIndexOrder[k+1], allIndexOrder[k]
|
||
|
||
|
||
|
||
reorderRes=[]
|
||
reorderInd=[]
|
||
|
||
|
||
# print('after')
|
||
# for index in allIndexOrder:
|
||
# print(index)
|
||
|
||
|
||
# 只记结果
|
||
|
||
repeat=[]
|
||
for i in range(len(allRes)):
|
||
if(allRes[i] not in repeat):
|
||
repeat.append(allRes[i])
|
||
reorderRes.append(allRes[i].tolist())
|
||
reorderInd.append(allIndexOrder[i].tolist())
|
||
|
||
|
||
orderLine1=self.GetGlobalOrderLine(indexOrder1.tolist())
|
||
orderLine2=self.GetGlobalOrderLine(indexOrder2.tolist())
|
||
orderLine=orderLine1+orderLine2
|
||
|
||
print(orderLine)
|
||
|
||
return reorderRes,reorderInd,orderLine1,orderLine2
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
#进行变异操作
|
||
def Variate(self,indexOrder,iterations):
|
||
|
||
theOrder=indexOrder
|
||
for i in range(batchSize):
|
||
if iterations==0:
|
||
randChoice=randint(1,20)
|
||
else:
|
||
randChoice=randint(1,5)
|
||
|
||
if randChoice>=5:
|
||
# newOrder = np.zeros(len(theOrder[i]))
|
||
smaLoc = randint(0,self.sentenSize-1)
|
||
bigLoc = randint(0,self.sentenSize-1)
|
||
|
||
if smaLoc>bigLoc:
|
||
smaLoc,bigLoc = bigLoc,smaLoc
|
||
|
||
|
||
temp=theOrder[i][smaLoc]
|
||
theOrder[i][smaLoc:bigLoc]=theOrder[i][smaLoc+1:bigLoc+1]
|
||
theOrder[i][bigLoc]=temp
|
||
|
||
return theOrder
|
||
|
||
|
||
|
||
|
||
|
||
def OX(self,fatherIndexOrder):
|
||
childIndexOrder=np.zeros([batchSize,self.sentenSize])
|
||
|
||
|
||
|
||
for i in range(int(batchSize/2)):
|
||
# father1Loc=i*2
|
||
# father2Loc=i*2+1
|
||
father1Loc=i
|
||
father2Loc=int(i+batchSize/2)
|
||
randChoice=randint(1,20)
|
||
|
||
if randChoice==1:
|
||
childIndexOrder[father1Loc] = fatherIndexOrder[father1Loc]
|
||
childIndexOrder[father2Loc] = fatherIndexOrder[father2Loc]
|
||
else:
|
||
father1 = fatherIndexOrder[father1Loc]
|
||
father2 = fatherIndexOrder[father2Loc]
|
||
|
||
smaLoc = randint(0,self.sentenSize-1)
|
||
bigLoc = randint(0,self.sentenSize-1)
|
||
|
||
if smaLoc>bigLoc:
|
||
smaLoc,bigLoc = bigLoc,smaLoc
|
||
|
||
childIndexOrder[father1Loc][smaLoc:bigLoc+1]=father1[smaLoc:bigLoc+1]
|
||
|
||
childLoc=0
|
||
|
||
|
||
for num in father2:
|
||
if childLoc == smaLoc:
|
||
childLoc = bigLoc+1
|
||
if childLoc >= self.sentenSize:
|
||
break
|
||
|
||
if num in father1[smaLoc:bigLoc+1]:
|
||
continue
|
||
|
||
childIndexOrder[father1Loc][childLoc]=num
|
||
|
||
childLoc+=1
|
||
|
||
|
||
|
||
|
||
father1 = fatherIndexOrder[father2Loc]
|
||
father2 = fatherIndexOrder[father1Loc]
|
||
|
||
|
||
smaLoc = randint(0,self.sentenSize-1)
|
||
bigLoc = randint(0,self.sentenSize-1)
|
||
|
||
if smaLoc>bigLoc:
|
||
smaLoc,bigLoc = bigLoc,smaLoc
|
||
|
||
childIndexOrder[father2Loc][smaLoc:bigLoc+1]=father1[smaLoc:bigLoc+1]
|
||
|
||
childLoc=0
|
||
|
||
for num in father2:
|
||
if childLoc == smaLoc:
|
||
childLoc = bigLoc+1
|
||
if childLoc >= self.sentenSize:
|
||
break
|
||
|
||
if num in father1[smaLoc:bigLoc+1]:
|
||
continue
|
||
|
||
childIndexOrder[father2Loc][childLoc]=num
|
||
childLoc+=1
|
||
|
||
return childIndexOrder
|
||
|
||
|
||
|
||
|
||
|
||
#将index转化为input进行处理
|
||
def IndexToInput(self,indexOrder):
|
||
#记录所有的评论，这些评论是原来的文字
|
||
comment=[]
|
||
#comment保存的评论，每个字都是用字典中的数字组成的
|
||
|
||
for i in range(batchSize):
|
||
allsub=[]
|
||
for index in indexOrder[i]:
|
||
allsub.append(self.senList[int(index)])
|
||
# TOFix!!!!!!!
|
||
fullSent=' '.join(allsub)
|
||
|
||
|
||
comment.append(fullSent)
|
||
|
||
|
||
|
||
return comment
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def GetGlobalOrderLine(self,indexOrder):
|
||
orderLine=[]
|
||
# print('indx',indexOrder)
|
||
lenth=len(indexOrder[0])
|
||
|
||
threHold=9
|
||
|
||
for sen in range(batchSize-threHold):#检查所有的句子
|
||
|
||
for i in range(lenth):#选定某个句子后检查所有的部分
|
||
for j in range(i+2,lenth+1):
|
||
count=0
|
||
target=indexOrder[sen][i:j]
|
||
# print('tar',target)
|
||
for k in range(1,batchSize):
|
||
searchOrder=indexOrder[k]
|
||
for l in range(lenth+i-j):
|
||
# if indexOrder[i:j] == searchOrder[l:l-i+j]:
|
||
# print('tar',target)
|
||
# print('searc',searchOrder[l:l-i+j])
|
||
if (target == searchOrder[l:l-i+j]):
|
||
# print('searchOrd',searchOrder[l:l-i+j])
|
||
count+=1
|
||
break
|
||
|
||
#如果重复率大于10
|
||
if count>3 and count>(15-len(target)*3):
|
||
isRepeat=False
|
||
for line in orderLine:
|
||
d = [False for c in target if c not in line]
|
||
if not d:
|
||
isRepeat=True
|
||
break
|
||
|
||
d = [False for c in line if c not in target]
|
||
if not d:
|
||
orderLine.remove(line)
|
||
|
||
if not isRepeat:
|
||
orderLine.append(target)
|
||
|
||
else:
|
||
break
|
||
|
||
|
||
|
||
return orderLine
|
||
|
||
|
||
|
||
|
||
if __name__ == "__main__":
|
||
myinput="44210 50393 43088 31169 23567 27393 34057 36073 47778 41062 34351 34428 39507 39018 45137 49284 42159 38415 51972 "
|
||
# myinput=input("输入")
|
||
|
||
MyGetDiffer=GetDifferCovid(myinput,'clause','judge')
|
||
|
||
reorderRes,reorderInd,orderLine1,orderLine2=MyGetDiffer.GetDiffOrder()
|