openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Try fixing occupied too much memory issue.

This commit is contained in:
zhangzihao 2021-01-19 19:26:39 +08:00
parent 20efcc67af
commit 5790b5692c
7 changed files with 116 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
libsearch/file-utils.cpp
View File

@ -530,5 +530,11 @@ void FileUtils::getTxtContent(QString &path, QString &textcontent)
stream.setCodec(codec);
textcontent = stream.readAll().replace("\n","");
file.close();
encodedString.clear();
chardet = NULL;
stream.flush();
return;
}

10
libsearch/index/document.cpp
View File

@ -1,10 +1,6 @@
#include "document.h"
#include <QDebug>
Document::Document()
{
}
void Document::setData(QString data)
{
if(data.isEmpty())
@ -42,15 +38,21 @@ void Document::setUniqueTerm(QString term)
if(term.isEmpty())
return;
m_document.add_term(term.toStdString());
// m_unique_term = new QString(term);
m_unique_term = std::move(term);
}
std::string Document::getUniqueTerm()
{
// qDebug()<<"m_unique_term!"<<*m_unique_term;
// qDebug() << QString::fromStdString(m_unique_term.toStdString());
return m_unique_term.toStdString();
}
void Document::setIndexText(QStringList indexText)
{
// QStringList indexTextList = indexText;
// m_index_text = new QStringList(indexText);
m_index_text = std::move(indexText);
}

2
libsearch/index/document.h
View File

@ -9,7 +9,7 @@
class Document
{
public:
Document();
Document() = default;
~Document(){}
Document(const Document& other){
m_document = other.m_document;

42
libsearch/index/first-index.cpp
View File

@ -21,6 +21,8 @@ void handler(int){
// while (true);
}
#define NEW_QUEUE(a) a = new QQueue<QString>(); qDebug("---------------------------%s %s %s new at %d..",__FILE__,__FUNCTION__,#a,__LINE__);
//#define DELETE_QUEUE(a )
FirstIndex::FirstIndex(const QString& path) : Traverse_BFS(path)
{
@ -47,8 +49,8 @@ FirstIndex::FirstIndex(const QString& path) : Traverse_BFS(path)
}
this->q_index = new QQueue<QVector<QString>>();
this->q_content_index = new QQueue<QString>();
//this->q_content_index = new QQueue<QString>();
NEW_QUEUE(this->q_content_index);
// this->mlm = new MessageListManager();
}
@ -61,8 +63,10 @@ FirstIndex::~FirstIndex()
if(this->q_content_index)
delete this->q_content_index;
this->q_content_index = nullptr;
// delete this->p_indexGenerator;
// this->p_indexGenerator;
if (this->p_indexGenerator)
delete this->p_indexGenerator;
this->p_indexGenerator = nullptr;
qDebug() << "~FirstIndex end";
// delete this->mlm;
// this->mlm = nullptr;
}
@ -97,6 +101,12 @@ void FirstIndex::run(){
else{
this->p_indexGenerator = IndexGenerator::getInstance(false,this);
}
// this->q_content_index->enqueue(QString("/home/zhangzihao/Desktop/qwerty/四库全书.txt"));
// this->p_indexGenerator->creatAllIndex(this->q_content_index);
QSemaphore sem(5);
QMutex mutex1, mutex2, mutex3;
mutex1.lock();
@ -133,11 +143,28 @@ void FirstIndex::run(){
mutex1.unlock();
mutex2.unlock();
mutex3.unlock();
qDebug() << "first index end;";
// qDebug() << "first index end;";
//don't use it now!!!!
//MouseZhangZh
// this->~FirstIndex();
qDebug() << "~FirstIndex end;";
// qDebug() << "~FirstIndex end;";
if (this->q_index)
delete this->q_index;
this->q_index = nullptr;
if (this->q_content_index)
delete this->q_content_index;
this->q_content_index = nullptr;
if (this->p_indexGenerator)
delete this->p_indexGenerator;
this->p_indexGenerator = nullptr;
QThreadPool::globalInstance()->releaseThread();
QThreadPool::globalInstance()->waitForDone();
@ -145,6 +172,9 @@ void FirstIndex::run(){
sigset( SIGTERM, handler);
qDebug() << "sigset end!";
//quit() is shit!!!
// return;
// exit(0);
this->quit();
// this->wait();
}

95
libsearch/index/index-generator.cpp
View File

@ -1,5 +1,4 @@
#include <QFile>
#include <QStandardPaths>
#include <QFileInfo>
#include <QDebug>
#include <QtConcurrent>
@ -10,12 +9,12 @@
#include "global-settings.h"
#include "chinese-segmentation.h"
#include "construct-document.h"
#include <QStandardPaths>
using namespace std;
#define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/index_data").toStdString()
#define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/content_index_data").toStdString()
static IndexGenerator *global_instance = nullptr;
QMutex IndexGenerator::m_mutex;
QList<Document> *_doc_list_path;
@ -45,26 +44,22 @@ bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList)
HandlePathList(messageList);
try
{
m_indexer = new Xapian::TermGenerator();
m_indexer->set_database(*m_database_path);
// m_indexer = new Xapian::TermGenerator();
m_indexer.set_database(*m_database_path);
//可以实现拼写纠正
// m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING);
m_indexer->set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
m_indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
int count =0;
for(int i = 0;i < _doc_list_path->size(); i++)
{
insertIntoDatabase(_doc_list_path->at(i));
for (auto i : *_doc_list_path){
if(++count == 9000)
{
insertIntoDatabase(i);
if(++count > 8999){
count = 0;
m_database_path->commit();
}
}
m_database_path->commit();
if(m_indexer)
delete m_indexer;
}
catch(const Xapian::Error &e)
{
@ -76,8 +71,6 @@ bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList)
_doc_list_path->clear();
delete _doc_list_path;
_doc_list_path = nullptr;
// Q_EMIT this->transactionFinished();
return true;
}
//文件内容索引
@ -91,17 +84,13 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList)
try
{
int count =0;
for(int i = 0;i < size; i++)
{
insertIntoContentDatabase(_doc_list_content->at(0));
_doc_list_content->removeFirst();
if(++count == 1000)
{
for (auto i : *_doc_list_content){
insertIntoContentDatabase(i);
if(++count > 999){
count = 0;
m_database_content->commit();
}
}
}
m_database_content->commit();
}
catch(const Xapian::Error &e)
@ -110,10 +99,11 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList)
GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE,"1");
assert(false);
}
_doc_list_content->clear();
delete _doc_list_content;
_doc_list_content = nullptr;
}
// Q_EMIT this->transactionFinished();
Q_EMIT this->transactionFinished();
FileUtils::_index_status = FINISH_CREATING_INDEX;
return true;
@ -139,29 +129,28 @@ IndexGenerator::~IndexGenerator()
{
QMutexLocker locker(&m_mutex);
qDebug() << "~IndexGenerator";
m_database_path->close();
m_database_content->close();
if(m_database_path)
delete m_database_path;
m_database_path = nullptr;
if(m_database_content)
delete m_database_content;
m_database_content = nullptr;
if(m_index_map)
delete m_index_map;
m_index_map = nullptr;
global_instance = nullptr;
// if(m_index_map)
// delete m_index_map;
// m_index_map = nullptr;
// if(m_doc_list_path)
// delete m_doc_list_path;
// m_doc_list_path = nullptr;
// if(_doc_list_content)
// if(m_doc_list_content)
// delete m_doc_list_content;
// m_doc_list_content = nullptr;
if(m_index_data_path)
delete m_index_data_path;
m_index_data_path = nullptr;
if(m_indexer)
delete m_indexer;
m_indexer = nullptr;
// if(m_index_data_path)
// delete m_index_data_path;
// m_index_data_path = nullptr;
// if(m_indexer)
// delete m_indexer;
// m_indexer = nullptr;
GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2");
GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2");
GlobalSettings::getInstance()->setValue(INDEX_GENERATOR_NORMAL_EXIT, "2");
@ -170,15 +159,15 @@ IndexGenerator::~IndexGenerator()
qDebug() << "~IndexGenerator end";
}
void IndexGenerator::insertIntoDatabase(Document doc)
void IndexGenerator::insertIntoDatabase(Document& doc)
{
// qDebug()<< "--index start--";
Xapian::Document document = doc.getXapianDocument();
m_indexer->set_document(document);
m_indexer.set_document(document);
// qDebug()<<doc.getIndexText();
for(auto i : doc.getIndexText()){
m_indexer->index_text(i.toStdString());
m_indexer.index_text(i.toStdString());
}
Xapian::docid innerId= m_database_path->replace_document(doc.getUniqueTerm(),document);
@ -186,8 +175,8 @@ void IndexGenerator::insertIntoDatabase(Document doc)
// qDebug()<< "--index finish--";
return;
}
void IndexGenerator::insertIntoContentDatabase(Document doc)
//#define fun(a) a=new ;printf()
void IndexGenerator::insertIntoContentDatabase(Document& doc)
{
Xapian::docid innerId= m_database_content->replace_document(doc.getUniqueTerm(),doc.getXapianDocument());
// qDebug()<<"replace doc docid="<<static_cast<int>(innerId);
@ -224,6 +213,10 @@ void IndexGenerator::HandlePathList(QQueue<QVector<QString>> *messageList)
qDebug()<<_doc_list_path->size();
// QList<Document> docList = future.results();
// m_doc_list_path = new QList<Document>(docList);
// m_doc_list_path = std::move(future.results());
// qDebug()<<m_doc_list_path.size();
qDebug()<<"Finish HandlePathList!";
return;
@ -259,6 +252,10 @@ void IndexGenerator::HandlePathList(QQueue<QString> *messageList)
// QList<Document> docList = future.results();
// m_doc_list_content = new QList<Document>(docList);
qDebug()<<_doc_list_content->size();
// QList<Document> docList = future.results();
// m_doc_list_content = new QList<Document>(docList);
// m_doc_list_content = std::move(future.results());
// future.cancel();
qDebug()<<"Finish HandlePathList for content index!";
return;
@ -314,14 +311,17 @@ Document IndexGenerator::GenerateContentDocument(const QString &path)
{
// 构造文本索引的document
QString content;
QStringList tmp;
QVector<SKeyWord> term;
SKeyWord skw;
Document doc;
QString uniqueterm;
QString upTerm;
FileReader::getTextContent(path,content);
QString uniqueterm = QString::fromStdString(FileUtils::makeDocUterm(path));
QString upTerm = QString::fromStdString(FileUtils::makeDocUterm(path.section("/",0,-2,QString::SectionIncludeLeadingSep)));
QVector<SKeyWord> term = ChineseSegmentation::getInstance()->callSegement(content);
term = ChineseSegmentation::getInstance()->callSegement(content);
// QStringList term = content.split("");
Document doc;
doc.setData(content);
doc.setUniqueTerm(uniqueterm);
doc.addTerm(upTerm);
@ -402,9 +402,10 @@ QStringList IndexGenerator::IndexSearch(QString indexText)
std::string data = doc.get_data();
Xapian::weight docScoreWeight = it.get_weight();
Xapian::percent docScorePercent = it.get_percent();
QFileInfo *info = new QFileInfo(QString::fromStdString(data));
// QFileInfo *info = new QFileInfo(QString::fromStdString(data));
QFileInfo info(QString::fromStdString(data));
if(!info->exists())
if(!info.exists())
{
// pathTobeDelete->append(QString::fromStdString(data));
qDebug()<<QString::fromStdString(data)<<"is not exist!!";

15
libsearch/index/index-generator.h
View File

@ -12,7 +12,6 @@
//#include <QMetaObject>
#include "document.h"
#include "file-reader.h"
//#include "chinese-segmentation.h"
extern QList<Document> *_doc_list_path;
extern QMutex _mutex_doc_list_path;
@ -48,18 +47,18 @@ private:
static Document GenerateDocument(const QVector<QString> &list);
static Document GenerateContentDocument(const QString &list);
//add one data in database
void insertIntoDatabase(Document doc);
void insertIntoContentDatabase(Document doc);
void insertIntoDatabase(Document& doc);
void insertIntoContentDatabase(Document& doc);
QMap<QString,QStringList> *m_index_map;
// QList<Document> *m_doc_list_path; //for path index
// QList<Document> *m_doc_list_content; // for text content index
QString *m_index_data_path;
Xapian::WritableDatabase *m_database_path;
Xapian::WritableDatabase *m_database_content;
QMap<QString,QStringList> m_index_map;
QString m_index_data_path;
Xapian::WritableDatabase* m_database_path;
Xapian::WritableDatabase* m_database_content;
std::string m_docstr;
std::string m_index_text_str;
Xapian::TermGenerator *m_indexer;
Xapian::TermGenerator m_indexer;
};
#endif // INDEXGENERATOR_H

12
src/main.cpp
View File

@ -168,11 +168,23 @@ int main(int argc, char *argv[])
// FirstIndex* fi = new FirstIndex("/home");
// fi->start();
qDebug() << "main start";
QThreadPool::globalInstance()->setExpiryTimeout(5);
// QThreadPool::globalInstance()->clear();
// setAutoDelete(true);
// FirstIndex fi("/home/zhangzihao/Desktop/qwerty");
// FirstIndex* fi = new FirstIndex("/home/zhangzihao/Desktop/qwerty");
FirstIndex fi("/home");
fi.start();
// fi->wait();
// fi->exit();
// delete fi;
// assert(false);
InotifyIndex* ii = InotifyIndex::getInstance("/home");
// InotifyIndex ii("/home");
ii->start();
return app.exec();
}