diff --git a/libsearch/file-utils.cpp b/libsearch/file-utils.cpp index 7fffb5f..e0fc868 100644 --- a/libsearch/file-utils.cpp +++ b/libsearch/file-utils.cpp @@ -530,5 +530,11 @@ void FileUtils::getTxtContent(QString &path, QString &textcontent) stream.setCodec(codec); textcontent = stream.readAll().replace("\n",""); + + file.close(); + encodedString.clear(); + chardet = NULL; + stream.flush(); + return; } diff --git a/libsearch/index/document.cpp b/libsearch/index/document.cpp index 359ac7f..98c0557 100644 --- a/libsearch/index/document.cpp +++ b/libsearch/index/document.cpp @@ -1,10 +1,6 @@ #include "document.h" #include -Document::Document() -{ -} - void Document::setData(QString data) { if(data.isEmpty()) @@ -42,15 +38,21 @@ void Document::setUniqueTerm(QString term) if(term.isEmpty()) return; m_document.add_term(term.toStdString()); + +// m_unique_term = new QString(term); m_unique_term = std::move(term); } std::string Document::getUniqueTerm() { +// qDebug()<<"m_unique_term!"<<*m_unique_term; +// qDebug() << QString::fromStdString(m_unique_term.toStdString()); return m_unique_term.toStdString(); } void Document::setIndexText(QStringList indexText) { +// QStringList indexTextList = indexText; +// m_index_text = new QStringList(indexText); m_index_text = std::move(indexText); } diff --git a/libsearch/index/document.h b/libsearch/index/document.h index 48a0111..10922d0 100644 --- a/libsearch/index/document.h +++ b/libsearch/index/document.h @@ -9,7 +9,7 @@ class Document { public: - Document(); + Document() = default; ~Document(){} Document(const Document& other){ m_document = other.m_document; diff --git a/libsearch/index/first-index.cpp b/libsearch/index/first-index.cpp index 93bf56e..bac677d 100644 --- a/libsearch/index/first-index.cpp +++ b/libsearch/index/first-index.cpp @@ -21,6 +21,8 @@ void handler(int){ // while (true); } +#define NEW_QUEUE(a) a = new QQueue(); qDebug("---------------------------%s %s %s new at %d..",__FILE__,__FUNCTION__,#a,__LINE__); +//#define DELETE_QUEUE(a ) FirstIndex::FirstIndex(const QString& path) : Traverse_BFS(path) { @@ -47,8 +49,8 @@ FirstIndex::FirstIndex(const QString& path) : Traverse_BFS(path) } this->q_index = new QQueue>(); - this->q_content_index = new QQueue(); - + //this->q_content_index = new QQueue(); + NEW_QUEUE(this->q_content_index); // this->mlm = new MessageListManager(); } @@ -61,8 +63,10 @@ FirstIndex::~FirstIndex() if(this->q_content_index) delete this->q_content_index; this->q_content_index = nullptr; -// delete this->p_indexGenerator; -// this->p_indexGenerator; + if (this->p_indexGenerator) + delete this->p_indexGenerator; + this->p_indexGenerator = nullptr; + qDebug() << "~FirstIndex end"; // delete this->mlm; // this->mlm = nullptr; } @@ -97,6 +101,12 @@ void FirstIndex::run(){ else{ this->p_indexGenerator = IndexGenerator::getInstance(false,this); } + +// this->q_content_index->enqueue(QString("/home/zhangzihao/Desktop/qwerty/四库全书.txt")); + +// this->p_indexGenerator->creatAllIndex(this->q_content_index); + + QSemaphore sem(5); QMutex mutex1, mutex2, mutex3; mutex1.lock(); @@ -133,11 +143,28 @@ void FirstIndex::run(){ mutex1.unlock(); mutex2.unlock(); mutex3.unlock(); - qDebug() << "first index end;"; + + + +// qDebug() << "first index end;"; //don't use it now!!!! //MouseZhangZh // this->~FirstIndex(); - qDebug() << "~FirstIndex end;"; +// qDebug() << "~FirstIndex end;"; + + + if (this->q_index) + delete this->q_index; + this->q_index = nullptr; + if (this->q_content_index) + delete this->q_content_index; + this->q_content_index = nullptr; + if (this->p_indexGenerator) + delete this->p_indexGenerator; + this->p_indexGenerator = nullptr; + + QThreadPool::globalInstance()->releaseThread(); + QThreadPool::globalInstance()->waitForDone(); @@ -145,6 +172,9 @@ void FirstIndex::run(){ sigset( SIGTERM, handler); qDebug() << "sigset end!"; + //quit() is shit!!! +// return; +// exit(0); this->quit(); // this->wait(); } diff --git a/libsearch/index/index-generator.cpp b/libsearch/index/index-generator.cpp index ca20663..fb66176 100644 --- a/libsearch/index/index-generator.cpp +++ b/libsearch/index/index-generator.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -10,12 +9,12 @@ #include "global-settings.h" #include "chinese-segmentation.h" #include "construct-document.h" +#include -using namespace std; - #define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/index_data").toStdString() #define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/content_index_data").toStdString() + static IndexGenerator *global_instance = nullptr; QMutex IndexGenerator::m_mutex; QList *_doc_list_path; @@ -45,26 +44,22 @@ bool IndexGenerator::creatAllIndex(QQueue > *messageList) HandlePathList(messageList); try { - m_indexer = new Xapian::TermGenerator(); - m_indexer->set_database(*m_database_path); +// m_indexer = new Xapian::TermGenerator(); + m_indexer.set_database(*m_database_path); //可以实现拼写纠正 // m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING); - m_indexer->set_stemming_strategy(Xapian::TermGenerator::STEM_SOME); + m_indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME); int count =0; - for(int i = 0;i < _doc_list_path->size(); i++) - { - insertIntoDatabase(_doc_list_path->at(i)); + for (auto i : *_doc_list_path){ - if(++count == 9000) - { + insertIntoDatabase(i); + if(++count > 8999){ count = 0; m_database_path->commit(); } } m_database_path->commit(); - if(m_indexer) - delete m_indexer; } catch(const Xapian::Error &e) { @@ -76,8 +71,6 @@ bool IndexGenerator::creatAllIndex(QQueue > *messageList) _doc_list_path->clear(); delete _doc_list_path; _doc_list_path = nullptr; -// Q_EMIT this->transactionFinished(); - return true; } //文件内容索引 @@ -91,17 +84,13 @@ bool IndexGenerator::creatAllIndex(QQueue *messageList) try { int count =0; - for(int i = 0;i < size; i++) - { - insertIntoContentDatabase(_doc_list_content->at(0)); - _doc_list_content->removeFirst(); - - if(++count == 1000) - { + for (auto i : *_doc_list_content){ + insertIntoContentDatabase(i); + if(++count > 999){ count = 0; m_database_content->commit(); } - } + } m_database_content->commit(); } catch(const Xapian::Error &e) @@ -110,10 +99,11 @@ bool IndexGenerator::creatAllIndex(QQueue *messageList) GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE,"1"); assert(false); } + _doc_list_content->clear(); delete _doc_list_content; _doc_list_content = nullptr; } -// Q_EMIT this->transactionFinished(); + Q_EMIT this->transactionFinished(); FileUtils::_index_status = FINISH_CREATING_INDEX; return true; @@ -139,29 +129,28 @@ IndexGenerator::~IndexGenerator() { QMutexLocker locker(&m_mutex); qDebug() << "~IndexGenerator"; - m_database_path->close(); - m_database_content->close(); if(m_database_path) delete m_database_path; m_database_path = nullptr; if(m_database_content) delete m_database_content; m_database_content = nullptr; - if(m_index_map) - delete m_index_map; - m_index_map = nullptr; + global_instance = nullptr; +// if(m_index_map) +// delete m_index_map; +// m_index_map = nullptr; // if(m_doc_list_path) // delete m_doc_list_path; // m_doc_list_path = nullptr; -// if(_doc_list_content) +// if(m_doc_list_content) // delete m_doc_list_content; // m_doc_list_content = nullptr; - if(m_index_data_path) - delete m_index_data_path; - m_index_data_path = nullptr; - if(m_indexer) - delete m_indexer; - m_indexer = nullptr; +// if(m_index_data_path) +// delete m_index_data_path; +// m_index_data_path = nullptr; +// if(m_indexer) +// delete m_indexer; +// m_indexer = nullptr; GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2"); GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2"); GlobalSettings::getInstance()->setValue(INDEX_GENERATOR_NORMAL_EXIT, "2"); @@ -170,15 +159,15 @@ IndexGenerator::~IndexGenerator() qDebug() << "~IndexGenerator end"; } -void IndexGenerator::insertIntoDatabase(Document doc) +void IndexGenerator::insertIntoDatabase(Document& doc) { // qDebug()<< "--index start--"; Xapian::Document document = doc.getXapianDocument(); - m_indexer->set_document(document); + m_indexer.set_document(document); // qDebug()<index_text(i.toStdString()); + m_indexer.index_text(i.toStdString()); } Xapian::docid innerId= m_database_path->replace_document(doc.getUniqueTerm(),document); @@ -186,8 +175,8 @@ void IndexGenerator::insertIntoDatabase(Document doc) // qDebug()<< "--index finish--"; return; } - -void IndexGenerator::insertIntoContentDatabase(Document doc) +//#define fun(a) a=new ;printf() +void IndexGenerator::insertIntoContentDatabase(Document& doc) { Xapian::docid innerId= m_database_content->replace_document(doc.getUniqueTerm(),doc.getXapianDocument()); // qDebug()<<"replace doc docid="<(innerId); @@ -224,6 +213,10 @@ void IndexGenerator::HandlePathList(QQueue> *messageList) qDebug()<<_doc_list_path->size(); +// QList docList = future.results(); +// m_doc_list_path = new QList(docList); +// m_doc_list_path = std::move(future.results()); +// qDebug()< *messageList) // QList docList = future.results(); // m_doc_list_content = new QList(docList); qDebug()<<_doc_list_content->size(); +// QList docList = future.results(); +// m_doc_list_content = new QList(docList); +// m_doc_list_content = std::move(future.results()); +// future.cancel(); qDebug()<<"Finish HandlePathList for content index!"; return; @@ -314,14 +311,17 @@ Document IndexGenerator::GenerateContentDocument(const QString &path) { // 构造文本索引的document QString content; + QStringList tmp; + QVector term; + SKeyWord skw; + Document doc; + QString uniqueterm; + QString upTerm; FileReader::getTextContent(path,content); - QString uniqueterm = QString::fromStdString(FileUtils::makeDocUterm(path)); - QString upTerm = QString::fromStdString(FileUtils::makeDocUterm(path.section("/",0,-2,QString::SectionIncludeLeadingSep))); - QVector term = ChineseSegmentation::getInstance()->callSegement(content); + term = ChineseSegmentation::getInstance()->callSegement(content); // QStringList term = content.split(""); - Document doc; doc.setData(content); doc.setUniqueTerm(uniqueterm); doc.addTerm(upTerm); @@ -402,9 +402,10 @@ QStringList IndexGenerator::IndexSearch(QString indexText) std::string data = doc.get_data(); Xapian::weight docScoreWeight = it.get_weight(); Xapian::percent docScorePercent = it.get_percent(); - QFileInfo *info = new QFileInfo(QString::fromStdString(data)); +// QFileInfo *info = new QFileInfo(QString::fromStdString(data)); + QFileInfo info(QString::fromStdString(data)); - if(!info->exists()) + if(!info.exists()) { // pathTobeDelete->append(QString::fromStdString(data)); qDebug()< #include "document.h" #include "file-reader.h" -//#include "chinese-segmentation.h" extern QList *_doc_list_path; extern QMutex _mutex_doc_list_path; @@ -48,18 +47,18 @@ private: static Document GenerateDocument(const QVector &list); static Document GenerateContentDocument(const QString &list); //add one data in database - void insertIntoDatabase(Document doc); - void insertIntoContentDatabase(Document doc); + void insertIntoDatabase(Document& doc); + void insertIntoContentDatabase(Document& doc); - QMap *m_index_map; // QList *m_doc_list_path; //for path index // QList *m_doc_list_content; // for text content index - QString *m_index_data_path; - Xapian::WritableDatabase *m_database_path; - Xapian::WritableDatabase *m_database_content; + QMap m_index_map; + QString m_index_data_path; + Xapian::WritableDatabase* m_database_path; + Xapian::WritableDatabase* m_database_content; std::string m_docstr; std::string m_index_text_str; - Xapian::TermGenerator *m_indexer; + Xapian::TermGenerator m_indexer; }; #endif // INDEXGENERATOR_H diff --git a/src/main.cpp b/src/main.cpp index 1abd4b0..1d03b8a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -168,11 +168,23 @@ int main(int argc, char *argv[]) // FirstIndex* fi = new FirstIndex("/home"); // fi->start(); qDebug() << "main start"; + + QThreadPool::globalInstance()->setExpiryTimeout(5); +// QThreadPool::globalInstance()->clear(); +// setAutoDelete(true); + +// FirstIndex fi("/home/zhangzihao/Desktop/qwerty"); +// FirstIndex* fi = new FirstIndex("/home/zhangzihao/Desktop/qwerty"); FirstIndex fi("/home"); fi.start(); +// fi->wait(); +// fi->exit(); +// delete fi; +// assert(false); InotifyIndex* ii = InotifyIndex::getInstance("/home"); // InotifyIndex ii("/home"); ii->start(); + return app.exec(); }