openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Merge pull request #246 from rookie-J/0506-dev 

Optimize xml-file treatment; Optimize jieba escape character;
This commit is contained in:
iaom 2021-05-31 14:18:12 +08:00 committed by GitHub
parent 923f913ad2 e01552618d
commit bfb1d518f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 138 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libchinese-segmentation/chinese-segmentation.cpp
View File

@ -77,10 +77,8 @@ std::vector<cppjieba::KeywordExtractor::Word> ChineseSegmentation::callSegementS
const size_t topk = -1;
std::vector<cppjieba::KeywordExtractor::Word> keywordres;
ChineseSegmentation::m_jieba->extractor.Extract(str, keywordres, topk);
// std::string().swap(s);
return keywordres;
}
void ChineseSegmentation::convert(std::vector<cppjieba::KeywordExtractor::Word> &keywordres, QVector<SKeyWord> &kw) {

8
libchinese-segmentation/cppjieba/DatTrie.hpp
View File

@ -103,11 +103,15 @@ public:
res.clear();
res.resize(end - begin);
const string text_str = EncodeRunesToString(begin, end);
for (size_t i = 0, begin_pos = 0; i < size_t(end - begin); i++) {
string text_str;
EncodeRunesToString(begin, end, text_str);
static const size_t max_num = 128;
JiebaDAT::result_pair_type result_pairs[max_num] = {};
for (size_t i = 0, begin_pos = 0; i < size_t(end - begin); i++) {
std::size_t num_results = dat_.commonPrefixSearch(&text_str[begin_pos], &result_pairs[0], max_num);
res[i].nexts.push_back(pair<size_t, const DatMemElem *>(i + 1, nullptr));

3
libchinese-segmentation/cppjieba/PreFilter.hpp
View File

@ -26,7 +26,8 @@ public:
WordRange range(cursor_, cursor_);
while (cursor_ != sentence_.end()) {
if (IsIn(symbols_, cursor_->rune)) {
//if (IsIn(symbols_, cursor_->rune)) {
if (cursor_->rune == 0x20) {
if (range.left == cursor_) {
cursor_ ++;
}

6
libchinese-segmentation/cppjieba/Unicode.hpp
View File

@ -195,6 +195,12 @@ inline string EncodeRunesToString(RuneStrArray::const_iterator begin, RuneStrArr
return str;
}
inline void EncodeRunesToString(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, string& str) {
RunePtrWrapper it_begin(begin), it_end(end);
limonp::Unicode32ToUtf8(it_begin, it_end, str);
return;
}
class Unicode32Counter {
public :
size_t length = 0;

67
libsearch/file-utils.cpp
View File

@ -20,6 +20,7 @@
*
*/
#include "file-utils.h"
#include <QXmlStreamReader>
using namespace Zeeker;
size_t FileUtils::_max_index_count = 0;
@ -488,6 +489,22 @@ void FileUtils::getDocxTextContent(QString &path, QString &textcontent) {
fileR.open(QIODevice::ReadOnly); //读取方式打开
QXmlStreamReader reader(&fileR);
while (!reader.atEnd()){
if(reader.readNextStartElement() and reader.name().toString() == "t"){
textcontent.append(reader.readElementText().replace("\n", "").replace("\r", " "));
if(textcontent.length() >= MAX_CONTENT_LENGTH/3){
break;
}
}
}
fileR.close();
file.close();
return;
/* //原加载DOM文档方式；
QDomDocument doc;
doc.setContent(fileR.readAll());
fileR.close();
@ -512,6 +529,7 @@ void FileUtils::getDocxTextContent(QString &path, QString &textcontent) {
}
file.close();
return;
*/
}
void FileUtils::getPptxTextContent(QString &path, QString &textcontent) {
@ -529,6 +547,31 @@ void FileUtils::getPptxTextContent(QString &path, QString &textcontent) {
}
if(fileList.isEmpty())
return;
for(int i = 0; i < fileList.size(); ++i){
QString name = prefix + QString::number(i + 1) + ".xml";
if(!file.setCurrentFile(name)) {
continue;
}
QuaZipFile fileR(&file);
fileR.open(QIODevice::ReadOnly);
QXmlStreamReader reader(&fileR);
while (!reader.atEnd()){
if(reader.readNextStartElement() and reader.name().toString() == "t"){
textcontent.append(reader.readElementText().replace("\n", "").replace("\r", " "));
if(textcontent.length() >= MAX_CONTENT_LENGTH/3){
break;
}
}
}
fileR.close();
}
file.close();
return;
/*
QDomElement sptree;
QDomElement sp;
QDomElement txbody;
@ -596,6 +639,7 @@ void FileUtils::getPptxTextContent(QString &path, QString &textcontent) {
}
file.close();
return;
*/
}
void FileUtils::getXlsxTextContent(QString &path, QString &textcontent) {
@ -610,8 +654,24 @@ void FileUtils::getXlsxTextContent(QString &path, QString &textcontent) {
return;
QuaZipFile fileR(&file);
fileR.open(QIODevice::ReadOnly); //读取方式打开
fileR.open(QIODevice::ReadOnly);
QXmlStreamReader reader(&fileR);
while (!reader.atEnd()){
if(reader.readNextStartElement() and reader.name().toString() == "t"){
textcontent.append(reader.readElementText().replace("\n", "").replace("\r", " "));
if(textcontent.length() >= MAX_CONTENT_LENGTH/3){
break;
}
}
}
fileR.close();
file.close();
return;
/*
QDomDocument doc;
doc.setContent(fileR.readAll());
fileR.close();
@ -641,6 +701,7 @@ void FileUtils::getXlsxTextContent(QString &path, QString &textcontent) {
}
file.close();
return;
*/
}
void FileUtils::getPdfTextContent(QString &path, QString &textcontent) {
@ -650,7 +711,7 @@ void FileUtils::getPdfTextContent(QString &path, QString &textcontent) {
const QRectF qf;
int pageNum = doc->numPages();
for(int i = 0; i < pageNum; ++i) {
textcontent.append(doc->page(i)->text(qf).replace("\n", ""));
textcontent.append(doc->page(i)->text(qf).replace("\n", "").replace("\r", " "));
if(textcontent.length() >= MAX_CONTENT_LENGTH / 3)
break;
}
@ -679,7 +740,7 @@ void FileUtils::getTxtContent(QString &path, QString &textcontent) {
stream.setCodec(codec);
uchardet_delete(chardet);
textcontent = stream.readAll().replace("\n", "");
textcontent = stream.readAll().replace("\n", "").replace("\r", " ");
file.close();
encodedString.clear();

12
libsearch/index/construct-document.cpp
View File

@ -110,16 +110,18 @@ void ConstructDocumentForContent::run() {
return;
QString uniqueterm = QString::fromStdString(FileUtils::makeDocUterm(m_path));
QString upTerm = QString::fromStdString(FileUtils::makeDocUterm(m_path.section("/", 0, -2, QString::SectionIncludeLeadingSep)));
// QVector<SKeyWord> term = ChineseSegmentation::getInstance()->callSegement(content.left(20480000));
//修改函数返回类型，修改入参为std::string引用--jxx20210519
std::vector<cppjieba::KeywordExtractor::Word> term = ChineseSegmentation::getInstance()->callSegementStd(content.left(20480000).toStdString());
Document doc;
doc.setData(content);
doc.setUniqueTerm(uniqueterm);
doc.addTerm(upTerm);
doc.addValue(m_path);
content = content.replace("\t", " ").replace("\xEF\xBC\x8C", " ").replace("\xE3\x80\x82", " ");
// QVector<SKeyWord> term = ChineseSegmentation::getInstance()->callSegement(content.left(20480000));
//修改函数返回类型，修改入参为std::string引用--jxx20210519
std::vector<cppjieba::KeywordExtractor::Word> term = ChineseSegmentation::getInstance()->callSegementStd(content.left(20480000).toStdString());
for(size_t i = 0; i < term.size(); ++i) {
doc.addPosting(term.at(i).word, term.at(i).offsets, static_cast<int>(term.at(i).weight));
}

3
libsearch/index/file-reader.cpp
View File

@ -31,8 +31,9 @@ void FileReader::getTextContent(QString path, QString &textContent) {
QFileInfo file(path);
QString strsfx = file.suffix();
if(name == "application/zip") {
if(strsfx.endsWith("docx"))
if(strsfx.endsWith("docx")){
FileUtils::getDocxTextContent(path, textContent);
}
if(strsfx.endsWith("pptx"))
FileUtils::getPptxTextContent(path, textContent);
if(strsfx.endsWith("xlsx"))

46
libsearch/index/first-index.cpp
View File

@ -47,8 +47,54 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
this->q_index->enqueue(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString((fileInfo.isDir() && (!fileInfo.isSymLink())) ? "1" : "0"));
if((fileInfo.fileName().split(".", QString::SkipEmptyParts).length() > 1) && (true == targetFileTypeMap[fileInfo.fileName().split(".").last()])) {
//this->q_content_index->enqueue(fileInfo.absoluteFilePath());
if(fileInfo.fileName().split(".").last() == "docx"){
QuaZip file(fileInfo.absoluteFilePath());
if(!file.open(QuaZip::mdUnzip))
return;
if(!file.setCurrentFile("word/document.xml", QuaZip::csSensitive))
return;
QuaZipFile fileR(&file);
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//docx解压缩后的xml文件为实际需要解析文件大小
qDebug() << "文件路径:" <<fileInfo.absoluteFilePath();
qDebug() << "文件大小:" << fileR.usize();
file.close();
}else if(fileInfo.fileName().split(".").last() == "pptx"){
QuaZip file(fileInfo.absoluteFilePath());
if(!file.open(QuaZip::mdUnzip))
return;
QString prefix("ppt/slides/slide");
qint64 fileSize(0);
qint64 fileIndex(0);
for(QString i : file.getFileNameList()) {
if(i.startsWith(prefix)){
QString name = prefix + QString::number(fileIndex + 1) + ".xml";
fileIndex++;
if(!file.setCurrentFile(name)) {
continue;
}
QuaZipFile fileR(&file);
fileSize += fileR.usize();
}
}
file.close();
qDebug() << "文件路径:" <<fileInfo.absoluteFilePath();
qDebug() << "文件大小:" << fileSize;
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileSize));//pptx解压缩后的xml文件为实际需要解析文件大小
}else if(fileInfo.fileName().split(".").last() == "xlsx"){
QuaZip file(fileInfo.absoluteFilePath());
if(!file.open(QuaZip::mdUnzip))
return;
if(!file.setCurrentFile("xl/sharedStrings.xml", QuaZip::csSensitive))
return;
QuaZipFile fileR(&file);
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//xlsx解压缩后的xml文件为实际解析文件大小
qDebug() << "文件路径:" <<fileInfo.absoluteFilePath();
qDebug() << "文件大小:" << fileR.usize();
file.close();
}else{
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size()));
}
}
}
void FirstIndex::run() {

6
libsearch/parser/binary-parser.cpp
View File

@ -4963,7 +4963,7 @@ bool KBinaryParser::read8DocText(FILE *pFile, const ppsInfoType *pPPS,
if(bUsesUnicode) {
ushort* usAucData = (ushort*)ptaucBytes;
content.append(QString::fromUtf16(usAucData).replace("\r", ""));
content.append(QString::fromUtf16(usAucData).replace("\n", "").replace("\r", " "));
usAucData = (ushort*)xfree((void*)usAucData);
ptaucBytes = NULL;
if(content.length() >= 682666) //20480000/3
@ -5066,7 +5066,7 @@ int KBinaryParser:: readSSTRecord(readDataParam &rdParam, ppsInfoType PPS_info,
} else {
ushort* usData = (ushort*)chData;
content.append(QString::fromUtf16(usData).replace("\r", ""));
content.append(QString::fromUtf16(usData).replace("\n", "").replace("\r", " "));
usData = (ushort*)xfree((void*)usData);
chData = NULL;
if(content.length() >= 682666) //20480000/3
@ -5131,7 +5131,7 @@ ULONG KBinaryParser::readPPtRecord(FILE* pFile, ppsInfoType* PPS_info, ULONG* au
return -1;
ushort* usData = (ushort*)chData;
content.append(QString::fromUtf16(usData).replace("\r", ""));
content.append(QString::fromUtf16(usData).replace("\n", "").replace("\r", " "));
usData = (ushort*)xfree((void*)usData);
chData = NULL;