openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Try fixing occupied too much memory issue.

This commit is contained in:
zhangzihao 2021-01-19 19:26:39 +08:00
parent 20efcc67af
commit 5790b5692c
7 changed files with 116 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
libsearch/file-utils.cpp
View File

@ -530,5 +530,11 @@ void FileUtils::getTxtContent(QString &path, QString &textcontent)
stream.setCodec(codec); stream.setCodec(codec);
textcontent = stream.readAll().replace("\n",""); textcontent = stream.readAll().replace("\n","");
file.close();
encodedString.clear();
chardet = NULL;
stream.flush();
return; return;
} }

10
libsearch/index/document.cpp
View File

@ -1,10 +1,6 @@
#include "document.h" #include "document.h"
#include <QDebug> #include <QDebug>
Document::Document()
{
}
void Document::setData(QString data) void Document::setData(QString data)
{ {
if(data.isEmpty()) if(data.isEmpty())
@ -42,15 +38,21 @@ void Document::setUniqueTerm(QString term)
if(term.isEmpty()) if(term.isEmpty())
return; return;
m_document.add_term(term.toStdString()); m_document.add_term(term.toStdString());
// m_unique_term = new QString(term);
m_unique_term = std::move(term); m_unique_term = std::move(term);
} }
std::string Document::getUniqueTerm() std::string Document::getUniqueTerm()
{ {
// qDebug()<<"m_unique_term!"<<*m_unique_term;
// qDebug() << QString::fromStdString(m_unique_term.toStdString());
return m_unique_term.toStdString(); return m_unique_term.toStdString();
} }
void Document::setIndexText(QStringList indexText) void Document::setIndexText(QStringList indexText)
{ {
// QStringList indexTextList = indexText;
// m_index_text = new QStringList(indexText);
m_index_text = std::move(indexText); m_index_text = std::move(indexText);
} }

2
libsearch/index/document.h
View File

@ -9,7 +9,7 @@
class Document class Document
{ {
public: public:
Document(); Document() = default;
~Document(){} ~Document(){}
Document(const Document& other){ Document(const Document& other){
m_document = other.m_document; m_document = other.m_document;

42
libsearch/index/first-index.cpp
View File

@ -21,6 +21,8 @@ void handler(int){
// while (true); // while (true);
} }
#define NEW_QUEUE(a) a = new QQueue<QString>(); qDebug("---------------------------%s %s %s new at %d..",__FILE__,__FUNCTION__,#a,__LINE__);
//#define DELETE_QUEUE(a )
FirstIndex::FirstIndex(const QString& path) : Traverse_BFS(path) FirstIndex::FirstIndex(const QString& path) : Traverse_BFS(path)
{ {
@ -47,8 +49,8 @@ FirstIndex::FirstIndex(const QString& path) : Traverse_BFS(path)
} }
this->q_index = new QQueue<QVector<QString>>(); this->q_index = new QQueue<QVector<QString>>();
this->q_content_index = new QQueue<QString>(); //this->q_content_index = new QQueue<QString>();
NEW_QUEUE(this->q_content_index);
// this->mlm = new MessageListManager(); // this->mlm = new MessageListManager();
} }
@ -61,8 +63,10 @@ FirstIndex::~FirstIndex()
if(this->q_content_index) if(this->q_content_index)
delete this->q_content_index; delete this->q_content_index;
this->q_content_index = nullptr; this->q_content_index = nullptr;
// delete this->p_indexGenerator; if (this->p_indexGenerator)
// this->p_indexGenerator; delete this->p_indexGenerator;
this->p_indexGenerator = nullptr;
qDebug() << "~FirstIndex end";
// delete this->mlm; // delete this->mlm;
// this->mlm = nullptr; // this->mlm = nullptr;
} }
@ -97,6 +101,12 @@ void FirstIndex::run(){
else{ else{
this->p_indexGenerator = IndexGenerator::getInstance(false,this); this->p_indexGenerator = IndexGenerator::getInstance(false,this);
} }
// this->q_content_index->enqueue(QString("/home/zhangzihao/Desktop/qwerty/四库全书.txt"));
// this->p_indexGenerator->creatAllIndex(this->q_content_index);
QSemaphore sem(5); QSemaphore sem(5);
QMutex mutex1, mutex2, mutex3; QMutex mutex1, mutex2, mutex3;
mutex1.lock(); mutex1.lock();
@ -133,11 +143,28 @@ void FirstIndex::run(){
mutex1.unlock(); mutex1.unlock();
mutex2.unlock(); mutex2.unlock();
mutex3.unlock(); mutex3.unlock();
qDebug() << "first index end;";
// qDebug() << "first index end;";
//don't use it now!!!! //don't use it now!!!!
//MouseZhangZh //MouseZhangZh
// this->~FirstIndex(); // this->~FirstIndex();
qDebug() << "~FirstIndex end;"; // qDebug() << "~FirstIndex end;";
if (this->q_index)
delete this->q_index;
this->q_index = nullptr;
if (this->q_content_index)
delete this->q_content_index;
this->q_content_index = nullptr;
if (this->p_indexGenerator)
delete this->p_indexGenerator;
this->p_indexGenerator = nullptr;
QThreadPool::globalInstance()->releaseThread();
QThreadPool::globalInstance()->waitForDone();
@ -145,6 +172,9 @@ void FirstIndex::run(){
sigset( SIGTERM, handler); sigset( SIGTERM, handler);
qDebug() << "sigset end!"; qDebug() << "sigset end!";
//quit() is shit!!!
// return;
// exit(0);
this->quit(); this->quit();
// this->wait(); // this->wait();
} }

93
libsearch/index/index-generator.cpp
View File

@ -1,5 +1,4 @@
#include <QFile> #include <QFile>
#include <QStandardPaths>
#include <QFileInfo> #include <QFileInfo>
#include <QDebug> #include <QDebug>
#include <QtConcurrent> #include <QtConcurrent>
@ -10,12 +9,12 @@
#include "global-settings.h" #include "global-settings.h"
#include "chinese-segmentation.h" #include "chinese-segmentation.h"
#include "construct-document.h" #include "construct-document.h"
#include <QStandardPaths>
using namespace std;
#define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/index_data").toStdString() #define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/index_data").toStdString()
#define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/content_index_data").toStdString() #define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/content_index_data").toStdString()
static IndexGenerator *global_instance = nullptr; static IndexGenerator *global_instance = nullptr;
QMutex IndexGenerator::m_mutex; QMutex IndexGenerator::m_mutex;
QList<Document> *_doc_list_path; QList<Document> *_doc_list_path;
@ -45,26 +44,22 @@ bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList)
HandlePathList(messageList); HandlePathList(messageList);
try try
{ {
m_indexer = new Xapian::TermGenerator(); // m_indexer = new Xapian::TermGenerator();
m_indexer->set_database(*m_database_path); m_indexer.set_database(*m_database_path);
//可以实现拼写纠正 //可以实现拼写纠正
// m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING); // m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING);
m_indexer->set_stemming_strategy(Xapian::TermGenerator::STEM_SOME); m_indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
int count =0; int count =0;
for(int i = 0;i < _doc_list_path->size(); i++) for (auto i : *_doc_list_path){
{
insertIntoDatabase(_doc_list_path->at(i));
if(++count == 9000) insertIntoDatabase(i);
{ if(++count > 8999){
count = 0; count = 0;
m_database_path->commit(); m_database_path->commit();
} }
} }
m_database_path->commit(); m_database_path->commit();
if(m_indexer)
delete m_indexer;
} }
catch(const Xapian::Error &e) catch(const Xapian::Error &e)
{ {
@ -76,8 +71,6 @@ bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList)
_doc_list_path->clear(); _doc_list_path->clear();
delete _doc_list_path; delete _doc_list_path;
_doc_list_path = nullptr; _doc_list_path = nullptr;
// Q_EMIT this->transactionFinished();
return true; return true;
} }
//文件内容索引 //文件内容索引
@ -91,13 +84,9 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList)
try try
{ {
int count =0; int count =0;
for(int i = 0;i < size; i++) for (auto i : *_doc_list_content){
{ insertIntoContentDatabase(i);
insertIntoContentDatabase(_doc_list_content->at(0)); if(++count > 999){
_doc_list_content->removeFirst();
if(++count == 1000)
{
count = 0; count = 0;
m_database_content->commit(); m_database_content->commit();
} }
@ -110,10 +99,11 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList)
GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE,"1"); GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE,"1");
assert(false); assert(false);
} }
_doc_list_content->clear();
delete _doc_list_content; delete _doc_list_content;
_doc_list_content = nullptr; _doc_list_content = nullptr;
} }
// Q_EMIT this->transactionFinished(); Q_EMIT this->transactionFinished();
FileUtils::_index_status = FINISH_CREATING_INDEX; FileUtils::_index_status = FINISH_CREATING_INDEX;
return true; return true;
@ -139,29 +129,28 @@ IndexGenerator::~IndexGenerator()
{ {
QMutexLocker locker(&m_mutex); QMutexLocker locker(&m_mutex);
qDebug() << "~IndexGenerator"; qDebug() << "~IndexGenerator";
m_database_path->close();
m_database_content->close();
if(m_database_path) if(m_database_path)
delete m_database_path; delete m_database_path;
m_database_path = nullptr; m_database_path = nullptr;
if(m_database_content) if(m_database_content)
delete m_database_content; delete m_database_content;
m_database_content = nullptr; m_database_content = nullptr;
if(m_index_map) global_instance = nullptr;
delete m_index_map; // if(m_index_map)
m_index_map = nullptr; // delete m_index_map;
// m_index_map = nullptr;
// if(m_doc_list_path) // if(m_doc_list_path)
// delete m_doc_list_path; // delete m_doc_list_path;
// m_doc_list_path = nullptr; // m_doc_list_path = nullptr;
// if(_doc_list_content) // if(m_doc_list_content)
// delete m_doc_list_content; // delete m_doc_list_content;
// m_doc_list_content = nullptr; // m_doc_list_content = nullptr;
if(m_index_data_path) // if(m_index_data_path)
delete m_index_data_path; // delete m_index_data_path;
m_index_data_path = nullptr; // m_index_data_path = nullptr;
if(m_indexer) // if(m_indexer)
delete m_indexer; // delete m_indexer;
m_indexer = nullptr; // m_indexer = nullptr;
GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2"); GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2");
GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2"); GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2");
GlobalSettings::getInstance()->setValue(INDEX_GENERATOR_NORMAL_EXIT, "2"); GlobalSettings::getInstance()->setValue(INDEX_GENERATOR_NORMAL_EXIT, "2");
@ -170,15 +159,15 @@ IndexGenerator::~IndexGenerator()
qDebug() << "~IndexGenerator end"; qDebug() << "~IndexGenerator end";
} }
void IndexGenerator::insertIntoDatabase(Document doc) void IndexGenerator::insertIntoDatabase(Document& doc)
{ {
// qDebug()<< "--index start--"; // qDebug()<< "--index start--";
Xapian::Document document = doc.getXapianDocument(); Xapian::Document document = doc.getXapianDocument();
m_indexer->set_document(document); m_indexer.set_document(document);
// qDebug()<<doc.getIndexText(); // qDebug()<<doc.getIndexText();
for(auto i : doc.getIndexText()){ for(auto i : doc.getIndexText()){
m_indexer->index_text(i.toStdString()); m_indexer.index_text(i.toStdString());
} }
Xapian::docid innerId= m_database_path->replace_document(doc.getUniqueTerm(),document); Xapian::docid innerId= m_database_path->replace_document(doc.getUniqueTerm(),document);
@ -186,8 +175,8 @@ void IndexGenerator::insertIntoDatabase(Document doc)
// qDebug()<< "--index finish--"; // qDebug()<< "--index finish--";
return; return;
} }
//#define fun(a) a=new ;printf()
void IndexGenerator::insertIntoContentDatabase(Document doc) void IndexGenerator::insertIntoContentDatabase(Document& doc)
{ {
Xapian::docid innerId= m_database_content->replace_document(doc.getUniqueTerm(),doc.getXapianDocument()); Xapian::docid innerId= m_database_content->replace_document(doc.getUniqueTerm(),doc.getXapianDocument());
// qDebug()<<"replace doc docid="<<static_cast<int>(innerId); // qDebug()<<"replace doc docid="<<static_cast<int>(innerId);
@ -224,6 +213,10 @@ void IndexGenerator::HandlePathList(QQueue<QVector<QString>> *messageList)
qDebug()<<_doc_list_path->size(); qDebug()<<_doc_list_path->size();
// QList<Document> docList = future.results();
// m_doc_list_path = new QList<Document>(docList);
// m_doc_list_path = std::move(future.results());
// qDebug()<<m_doc_list_path.size();
qDebug()<<"Finish HandlePathList!"; qDebug()<<"Finish HandlePathList!";
return; return;
@ -259,6 +252,10 @@ void IndexGenerator::HandlePathList(QQueue<QString> *messageList)
// QList<Document> docList = future.results(); // QList<Document> docList = future.results();
// m_doc_list_content = new QList<Document>(docList); // m_doc_list_content = new QList<Document>(docList);
qDebug()<<_doc_list_content->size(); qDebug()<<_doc_list_content->size();
// QList<Document> docList = future.results();
// m_doc_list_content = new QList<Document>(docList);
// m_doc_list_content = std::move(future.results());
// future.cancel();
qDebug()<<"Finish HandlePathList for content index!"; qDebug()<<"Finish HandlePathList for content index!";
return; return;
@ -314,14 +311,17 @@ Document IndexGenerator::GenerateContentDocument(const QString &path)
{ {
// 构造文本索引的document // 构造文本索引的document
QString content; QString content;
QStringList tmp;
QVector<SKeyWord> term;
SKeyWord skw;
Document doc;
QString uniqueterm;
QString upTerm;
FileReader::getTextContent(path,content); FileReader::getTextContent(path,content);
QString uniqueterm = QString::fromStdString(FileUtils::makeDocUterm(path));
QString upTerm = QString::fromStdString(FileUtils::makeDocUterm(path.section("/",0,-2,QString::SectionIncludeLeadingSep)));
QVector<SKeyWord> term = ChineseSegmentation::getInstance()->callSegement(content); term = ChineseSegmentation::getInstance()->callSegement(content);
// QStringList term = content.split(""); // QStringList term = content.split("");
Document doc;
doc.setData(content); doc.setData(content);
doc.setUniqueTerm(uniqueterm); doc.setUniqueTerm(uniqueterm);
doc.addTerm(upTerm); doc.addTerm(upTerm);
@ -402,9 +402,10 @@ QStringList IndexGenerator::IndexSearch(QString indexText)
std::string data = doc.get_data(); std::string data = doc.get_data();
Xapian::weight docScoreWeight = it.get_weight(); Xapian::weight docScoreWeight = it.get_weight();
Xapian::percent docScorePercent = it.get_percent(); Xapian::percent docScorePercent = it.get_percent();
QFileInfo *info = new QFileInfo(QString::fromStdString(data)); // QFileInfo *info = new QFileInfo(QString::fromStdString(data));
QFileInfo info(QString::fromStdString(data));
if(!info->exists()) if(!info.exists())
{ {
// pathTobeDelete->append(QString::fromStdString(data)); // pathTobeDelete->append(QString::fromStdString(data));
qDebug()<<QString::fromStdString(data)<<"is not exist!!"; qDebug()<<QString::fromStdString(data)<<"is not exist!!";

15
libsearch/index/index-generator.h
View File

@ -12,7 +12,6 @@
//#include <QMetaObject> //#include <QMetaObject>
#include "document.h" #include "document.h"
#include "file-reader.h" #include "file-reader.h"
//#include "chinese-segmentation.h"
extern QList<Document> *_doc_list_path; extern QList<Document> *_doc_list_path;
extern QMutex _mutex_doc_list_path; extern QMutex _mutex_doc_list_path;
@ -48,18 +47,18 @@ private:
static Document GenerateDocument(const QVector<QString> &list); static Document GenerateDocument(const QVector<QString> &list);
static Document GenerateContentDocument(const QString &list); static Document GenerateContentDocument(const QString &list);
//add one data in database //add one data in database
void insertIntoDatabase(Document doc); void insertIntoDatabase(Document& doc);
void insertIntoContentDatabase(Document doc); void insertIntoContentDatabase(Document& doc);
QMap<QString,QStringList> *m_index_map;
// QList<Document> *m_doc_list_path; //for path index // QList<Document> *m_doc_list_path; //for path index
// QList<Document> *m_doc_list_content; // for text content index // QList<Document> *m_doc_list_content; // for text content index
QString *m_index_data_path; QMap<QString,QStringList> m_index_map;
Xapian::WritableDatabase *m_database_path; QString m_index_data_path;
Xapian::WritableDatabase *m_database_content; Xapian::WritableDatabase* m_database_path;
Xapian::WritableDatabase* m_database_content;
std::string m_docstr; std::string m_docstr;
std::string m_index_text_str; std::string m_index_text_str;
Xapian::TermGenerator *m_indexer; Xapian::TermGenerator m_indexer;
}; };
#endif // INDEXGENERATOR_H #endif // INDEXGENERATOR_H

12
src/main.cpp
View File

@ -168,11 +168,23 @@ int main(int argc, char *argv[])
// FirstIndex* fi = new FirstIndex("/home"); // FirstIndex* fi = new FirstIndex("/home");
// fi->start(); // fi->start();
qDebug() << "main start"; qDebug() << "main start";
QThreadPool::globalInstance()->setExpiryTimeout(5);
// QThreadPool::globalInstance()->clear();
// setAutoDelete(true);
// FirstIndex fi("/home/zhangzihao/Desktop/qwerty");
// FirstIndex* fi = new FirstIndex("/home/zhangzihao/Desktop/qwerty");
FirstIndex fi("/home"); FirstIndex fi("/home");
fi.start(); fi.start();
// fi->wait();
// fi->exit();
// delete fi;
// assert(false);
InotifyIndex* ii = InotifyIndex::getInstance("/home"); InotifyIndex* ii = InotifyIndex::getInstance("/home");
// InotifyIndex ii("/home"); // InotifyIndex ii("/home");
ii->start(); ii->start();
return app.exec(); return app.exec();
} }