From 452896a354a248da74fd3b43d41e0813d6ddf14b Mon Sep 17 00:00:00 2001 From: zhangpengfei Date: Thu, 21 Jan 2021 13:50:21 +0800 Subject: [PATCH 1/3] Creat two sub-processes for file index to reduce the resource consumption(ongoing). --- debian/changelog | 11 +- .../chinese-segmentation.cpp | 2 +- libsearch/file-utils.h | 1 + libsearch/index/file-searcher.cpp | 3 + libsearch/index/first-index.cpp | 119 +++--- libsearch/index/first-index.h | 5 + libsearch/index/index-generator.cpp | 4 + libsearch/index/inotify-index.cpp | 355 ++++++++++-------- libsearch/index/inotify-index.h | 4 + src/main.cpp | 2 + 10 files changed, 289 insertions(+), 217 deletions(-) diff --git a/debian/changelog b/debian/changelog index b2e917e..c625d04 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +ukui-search (0.0.1+0120) v101; urgency=medium + + * Bugs Fixed. + + -- zhangpengfei Wed, 20 Jan 2021 15:55:57 +0800 + ukui-search (0.0.1+0118) v101; urgency=medium * Feature: Add animation into inputbox. @@ -11,8 +17,7 @@ ukui-search (0.0.1+0118) v101; urgency=medium * Fix: Translation is incompleted.(33047) * Fix: Widget crashed when searching & open file/filepath failed. - -- zhangjiaping Mon, 18 Jan 2021 14:3 -6:12 +0800 + -- zhangjiaping Mon, 18 Jan 2021 14:31:24 +0800 ukui-search (0.0.1+0115) v101; urgency=medium @@ -24,7 +29,7 @@ ukui-search (0.0.1+0114) v101; urgency=medium * Bugs fixed. - -- zhangpengfei Thu, 14 Jan 2021 20:47:42 +0800 + -- zhangpengfei Thu, 14 Jan 2021 20:47:42 +0800 ukui-search (0.0.1+0113) v101; urgency=medium diff --git a/libchinese-segmentation/chinese-segmentation.cpp b/libchinese-segmentation/chinese-segmentation.cpp index efbb8c8..2bcec40 100644 --- a/libchinese-segmentation/chinese-segmentation.cpp +++ b/libchinese-segmentation/chinese-segmentation.cpp @@ -6,7 +6,6 @@ QMutex ChineseSegmentation::m_mutex; ChineseSegmentation::ChineseSegmentation() { - QMutexLocker locker(&m_mutex); const char * const DICT_PATH = "/usr/share/ukui-search/res/dict/jieba.dict.utf8"; const char * const HMM_PATH = "/usr/share/ukui-search/res/dict/hmm_model.utf8"; const char * const USER_DICT_PATH ="/usr/share/ukui-search/res/dict/user.dict.utf8"; @@ -29,6 +28,7 @@ ChineseSegmentation::~ChineseSegmentation() ChineseSegmentation *ChineseSegmentation::getInstance() { + QMutexLocker locker(&m_mutex); if (!global_instance_chinese_segmentation) { global_instance_chinese_segmentation = new ChineseSegmentation; } diff --git a/libsearch/file-utils.h b/libsearch/file-utils.h index 3d99dbe..5594615 100644 --- a/libsearch/file-utils.h +++ b/libsearch/file-utils.h @@ -10,6 +10,7 @@ #define CREATING_INDEX 1 #define FINISH_CREATING_INDEX 2 + class LIBSEARCH_EXPORT FileUtils { public: diff --git a/libsearch/index/file-searcher.cpp b/libsearch/index/file-searcher.cpp index f2ee9c7..9377112 100644 --- a/libsearch/index/file-searcher.cpp +++ b/libsearch/index/file-searcher.cpp @@ -66,6 +66,7 @@ void FileSearcher::onKeywordSearch(QString keyword,QQueue *searchResult total += resultCount; begin += num; } + return; }); // Q_EMIT this->resultFile(m_search_result_file); //dir @@ -84,6 +85,7 @@ void FileSearcher::onKeywordSearch(QString keyword,QQueue *searchResult total += resultCount; begin += num; } + return; }); // Q_EMIT this->resultDir(m_search_result_dir); //content @@ -103,6 +105,7 @@ void FileSearcher::onKeywordSearch(QString keyword,QQueue *searchResult total += resultCount; begin += num; } + return; }); // Q_EMIT this->resultContent(m_search_result_content); } diff --git a/libsearch/index/first-index.cpp b/libsearch/index/first-index.cpp index 7feb60a..9ba13c9 100644 --- a/libsearch/index/first-index.cpp +++ b/libsearch/index/first-index.cpp @@ -2,7 +2,6 @@ #include "first-index.h" #include - void handler(int){ qDebug() << "Recieved SIGTERM!"; GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2"); @@ -107,67 +106,81 @@ void FirstIndex::run(){ // this->p_indexGenerator->creatAllIndex(this->q_content_index); - FileUtils::_index_status = CREATING_INDEX; - QSemaphore sem(5); - QMutex mutex1, mutex2, mutex3; - mutex1.lock(); - mutex2.lock(); - mutex3.lock(); - sem.acquire(4); - QtConcurrent::run([&](){ - sem.acquire(1); + pid_t pid; + pid = fork(); + if(pid == 0) + { + prctl(PR_SET_NAME,"first-index"); + FileUtils::_index_status = CREATING_INDEX; + QSemaphore sem(5); + QMutex mutex1, mutex2, mutex3; + mutex1.lock(); + mutex2.lock(); + mutex3.lock(); + sem.acquire(4); + QtConcurrent::run([&](){ + sem.acquire(1); + mutex1.unlock(); + this->Traverse(); + FileUtils::_max_index_count = this->q_index->length(); + sem.release(5); + }); + QtConcurrent::run([&](){ + sem.acquire(2); + mutex2.unlock(); + qDebug() << "index start;"; + this->p_indexGenerator->creatAllIndex(this->q_index); + qDebug() << "index end;"; + sem.release(2); + }); + QtConcurrent::run([&](){ + sem.acquire(2); + mutex3.unlock(); + qDebug() << "content index start;"; + this->p_indexGenerator->creatAllIndex(this->q_content_index); + qDebug() << "content index end;"; + sem.release(2); + }); + mutex1.lock(); + mutex2.lock(); + mutex3.lock(); + sem.acquire(5); mutex1.unlock(); - this->Traverse(); - FileUtils::_max_index_count = this->q_index->length(); - sem.release(5); - }); - QtConcurrent::run([&](){ - sem.acquire(2); mutex2.unlock(); - qDebug() << "index start;"; - this->p_indexGenerator->creatAllIndex(this->q_index); - qDebug() << "index end;"; - sem.release(2); - }); - QtConcurrent::run([&](){ - sem.acquire(2); mutex3.unlock(); - qDebug() << "content index start;"; - this->p_indexGenerator->creatAllIndex(this->q_content_index); - qDebug() << "content index end;"; - sem.release(2); - }); - mutex1.lock(); - mutex2.lock(); - mutex3.lock(); - sem.acquire(5); - mutex1.unlock(); - mutex2.unlock(); - mutex3.unlock(); + _exit(0); -// qDebug() << "first index end;"; - //don't use it now!!!! - //MouseZhangZh -// this->~FirstIndex(); -// qDebug() << "~FirstIndex end;"; + + // qDebug() << "first index end;"; + //don't use it now!!!! + //MouseZhangZh + // this->~FirstIndex(); + // qDebug() << "~FirstIndex end;" - if (this->q_index) - delete this->q_index; - this->q_index = nullptr; - if (this->q_content_index) - delete this->q_content_index; - this->q_content_index = nullptr; - if (this->p_indexGenerator) - delete this->p_indexGenerator; - this->p_indexGenerator = nullptr; - - QThreadPool::globalInstance()->releaseThread(); - QThreadPool::globalInstance()->waitForDone(); - + if (this->q_index) + delete this->q_index; + this->q_index = nullptr; + if (this->q_content_index) + delete this->q_content_index; + this->q_content_index = nullptr; + if (this->p_indexGenerator) + delete this->p_indexGenerator; + this->p_indexGenerator = nullptr; + QThreadPool::globalInstance()->releaseThread(); + QThreadPool::globalInstance()->waitForDone(); + } + else if(pid < 0) + { + qWarning()<<"First Index fork error!!"; + } + else + { + waitpid(pid,NULL,0); + } FileUtils::_index_status = FINISH_CREATING_INDEX; qDebug() << "sigset start!"; diff --git a/libsearch/index/first-index.h b/libsearch/index/first-index.h index 914fa85..794267d 100644 --- a/libsearch/index/first-index.h +++ b/libsearch/index/first-index.h @@ -4,6 +4,11 @@ #include #include #include +#include +#include +#include +#include +#include //#include #include "traverse_bfs.h" #include "global-settings.h" diff --git a/libsearch/index/index-generator.cpp b/libsearch/index/index-generator.cpp index 96c60a8..faad57d 100644 --- a/libsearch/index/index-generator.cpp +++ b/libsearch/index/index-generator.cpp @@ -450,7 +450,11 @@ bool IndexGenerator::deleteAllIndex(QStringList *pathlist) qDebug()<<"delete path"<commit(); + m_database_content->commit(); qDebug()<< "--delete finish--"; +// qDebug()<<"m_database_path->get_lastdocid()!!!"<get_lastdocid(); + +// qDebug()<<"m_database_path->get_doccount()!!!"<get_doccount(); } catch(const Xapian::Error &e) { diff --git a/libsearch/index/inotify-index.cpp b/libsearch/index/inotify-index.cpp index 642aa14..e87b75d 100644 --- a/libsearch/index/inotify-index.cpp +++ b/libsearch/index/inotify-index.cpp @@ -1,6 +1,5 @@ #include "inotify-index.h" - InotifyIndex::InotifyIndex(const QString& path) : Traverse_BFS(path) { /*-------------ukuisearchdbus Test start-----------------*/ @@ -88,51 +87,64 @@ bool InotifyIndex::RemoveWatch(const QString &path){ return true; } -/* - * Symbolic Link!!!!!!!!!!!!!!!!!! - * Sysmbolic link to database dir will make a Infinite loop !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - * MouseZhangZh -*/ +void InotifyIndex::eventProcess(const char* buf, ssize_t tmp){ + // qDebug() << "Read " << numRead << " bytes from inotify fd"; -void InotifyIndex::run(){ + /* Process all of the events in buffer returned by read() */ - char * p; - char buf[BUF_LEN] __attribute__((aligned(8))); - - ssize_t numRead; QQueue>* indexQueue = new QQueue>(); QQueue* contentIndexQueue = new QQueue(); - for (;;) { /* Read events forever */ - numRead = read(m_fd, buf, BUF_LEN); - if (numRead == 0) { - qDebug() << "read() from inotify fd returned 0!"; - } - if (numRead == -1) { - qDebug() << "read"; - } -// qDebug() << "Read " << numRead << " bytes from inotify fd"; + ssize_t numRead = 0; + numRead = tmp; + char * p = const_cast(buf); - /* Process all of the events in buffer returned by read() */ + for (; p < buf + numRead;) { + struct inotify_event * event = reinterpret_cast(p); +// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask; + if(event->name[0] != '.'){ + qDebug() << QString(currentPath[event->wd] + '/' + event->name); + FileUtils::_index_status = CREATING_INDEX; - for (p = buf; p < buf + numRead;) { - struct inotify_event * event = reinterpret_cast(p); -// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask; - if(event->name[0] != '.'){ - qDebug() << QString(currentPath[event->wd] + '/' + event->name); - FileUtils::_index_status = CREATING_INDEX; + // switch (event->mask) { + if (event->mask & IN_CREATE){ + if (event->mask & IN_ISDIR){ + AddWatch(currentPath[event->wd] + '/' + event->name); + this->setPath(currentPath[event->wd] + '/' + event->name); + Traverse(); + } -// switch (event->mask) { - if (event->mask & IN_CREATE){ - if (event->mask & IN_ISDIR){ - AddWatch(currentPath[event->wd] + '/' + event->name); - this->setPath(currentPath[event->wd] + '/' + event->name); - Traverse(); + /*--------------------------------*/ + // IndexGenerator::getInstance()->creatAllIndex(QQueue>(QVector() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0"))); + indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); + IndexGenerator::getInstance()->creatAllIndex(indexQueue); + indexQueue->clear(); + for (auto i : this->targetFileTypeVec){ + if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ + contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); + IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); + contentIndexQueue->clear(); + break; } + } + goto next; + } - /*--------------------------------*/ -// IndexGenerator::getInstance()->creatAllIndex(QQueue>(QVector() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0"))); + + if ((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){ + if (event->mask & IN_ISDIR){ + RemoveWatch(currentPath[event->wd] + '/' + event->name); + } + //delete once more + IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); + goto next; + } + + + if (event->mask & IN_MODIFY){ + if (!(event->mask & IN_ISDIR)){ + IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); IndexGenerator::getInstance()->creatAllIndex(indexQueue); indexQueue->clear(); @@ -144,134 +156,101 @@ void InotifyIndex::run(){ break; } } - goto next; } - - - if ((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){ - if (event->mask & IN_ISDIR){ - RemoveWatch(currentPath[event->wd] + '/' + event->name); - } - //delete once more - IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); - goto next; - } - - - if (event->mask & IN_MODIFY){ - if (!(event->mask & IN_ISDIR)){ - IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); - indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); - IndexGenerator::getInstance()->creatAllIndex(indexQueue); - indexQueue->clear(); - for (auto i : this->targetFileTypeVec){ - if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ - contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); - IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); - contentIndexQueue->clear(); - break; - } - } - } - goto next; - } - - - if (event->mask & IN_MOVED_TO){ - if (event->mask & IN_ISDIR){ - RemoveWatch(currentPath[event->wd] + '/' + event->name); - IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); - AddWatch(currentPath[event->wd] + '/' + event->name); - this->setPath(currentPath[event->wd] + '/' + event->name); - Traverse(); - - // - - - - indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); - IndexGenerator::getInstance()->creatAllIndex(indexQueue); - indexQueue->clear(); - for (auto i : this->targetFileTypeVec){ - if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ - contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); - IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); - contentIndexQueue->clear(); - break; - } - } - } - else { - IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); - indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); - IndexGenerator::getInstance()->creatAllIndex(indexQueue); - indexQueue->clear(); - for (auto i : this->targetFileTypeVec){ - if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ - contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); - IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); - contentIndexQueue->clear(); - break; - } - } - } - goto next; - } - -// } - -// //传创建或移动过来的文件路径 -// if((event->mask & IN_CREATE)){ -// //添加监视要先序遍历，先添加top节点 -// if (event->mask & IN_ISDIR){ -// AddWatch(currentPath[event->wd] + '/' + event->name); -// this->setPath(currentPath[event->wd] + '/' + event->name); -// Traverse(); -// } - -// /*--------------------------------*/ -//// IndexGenerator::getInstance()->creatAllIndex(QQueue>(QVector() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0"))); -// indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); -// IndexGenerator::getInstance()->creatAllIndex(indexQueue); -// indexQueue->clear(); -// for (auto i : this->targetFileTypeVec){ -// if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ -// contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); -// IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); -// contentIndexQueue->clear(); -// break; -// } -// } -// /*--------------------------------*/ -// } -// else if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){ -// if (event->mask & IN_ISDIR){ -// RemoveWatch(currentPath[event->wd] + '/' + event->name); -// } -// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); -// } -// else if((event->mask & IN_MODIFY) | (event->mask & IN_MOVED_TO)){ -// if (!(event->mask & IN_ISDIR)){ -// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); -// indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); -// IndexGenerator::getInstance()->creatAllIndex(indexQueue); -// indexQueue->clear(); -// for (auto i : this->targetFileTypeVec){ -// if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ -// contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); -// IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); -// contentIndexQueue->clear(); -// break; -// } -// } -// } -// } - /*--------------------------------*/ - FileUtils::_index_status = FINISH_CREATING_INDEX; + goto next; } -next: - p += sizeof(struct inotify_event) + event->len; + + + if (event->mask & IN_MOVED_TO){ + if (event->mask & IN_ISDIR){ + RemoveWatch(currentPath[event->wd] + '/' + event->name); + IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); + AddWatch(currentPath[event->wd] + '/' + event->name); + this->setPath(currentPath[event->wd] + '/' + event->name); + Traverse(); + + indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); + IndexGenerator::getInstance()->creatAllIndex(indexQueue); + indexQueue->clear(); + for (auto i : this->targetFileTypeVec){ + if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ + contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); + IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); + contentIndexQueue->clear(); + break; + } + } + } + else { + IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); + indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); + IndexGenerator::getInstance()->creatAllIndex(indexQueue); + indexQueue->clear(); + for (auto i : this->targetFileTypeVec){ + if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ + contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); + IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); + contentIndexQueue->clear(); + break; + } + } + } + goto next; + } + + // } + + // //传创建或移动过来的文件路径 + // if((event->mask & IN_CREATE)){ + // //添加监视要先序遍历，先添加top节点 + // if (event->mask & IN_ISDIR){ + // AddWatch(currentPath[event->wd] + '/' + event->name); + // this->setPath(currentPath[event->wd] + '/' + event->name); + // Traverse(); + // } + + // /*--------------------------------*/ + //// IndexGenerator::getInstance()->creatAllIndex(QQueue>(QVector() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0"))); + // indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); + // IndexGenerator::getInstance()->creatAllIndex(indexQueue); + // indexQueue->clear(); + // for (auto i : this->targetFileTypeVec){ + // if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ + // contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); + // IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); + // contentIndexQueue->clear(); + // break; + // } + // } + // /*--------------------------------*/ + // } + // else if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){ + // if (event->mask & IN_ISDIR){ + // RemoveWatch(currentPath[event->wd] + '/' + event->name); + // } + // IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); + // } + // else if((event->mask & IN_MODIFY) | (event->mask & IN_MOVED_TO)){ + // if (!(event->mask & IN_ISDIR)){ + // IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); + // indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); + // IndexGenerator::getInstance()->creatAllIndex(indexQueue); + // indexQueue->clear(); + // for (auto i : this->targetFileTypeVec){ + // if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ + // contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); + // IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); + // contentIndexQueue->clear(); + // break; + // } + // } + // } + // } + /*--------------------------------*/ + FileUtils::_index_status = FINISH_CREATING_INDEX; } +next: + p += sizeof(struct inotify_event) + event->len; } delete indexQueue; @@ -279,3 +258,59 @@ next: delete contentIndexQueue; contentIndexQueue = nullptr; } + +/* + * Symbolic Link!!!!!!!!!!!!!!!!!! + * Sysmbolic link to database dir will make a Infinite loop !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + * MouseZhangZh +*/ + +void InotifyIndex::run(){ + char buf[BUF_LEN] __attribute__((aligned(8))); + + ssize_t numRead; + + for (;;) { /* Read events forever */ + numRead = read(m_fd, buf, BUF_LEN); + + + pid_t pid; + pid = fork(); + if(pid == 0) + { + prctl(PR_SET_NAME,"inotify-index"); + if (numRead == 0) { + qDebug() << "read() from inotify fd returned 0!"; + } + if (numRead == -1) { + qDebug() << "read"; + } + eventProcess(buf, numRead); + QTimer* liveTime = new QTimer(this); + bool b_timeout = false; + liveTime->setInterval(30000); + connect(liveTime, &QTimer::timeout, this, [&](){ +// _exit(0); + b_timeout = true; + }); + liveTime->start(); + for (;;){ + numRead = read(m_fd, buf, BUF_LEN); + liveTime->stop(); + this->eventProcess(buf, numRead); + if (b_timeout){ + _exit(0); + } + liveTime->start(); + } + } + else if (pid > 0){ + memset(buf, 0x00, BUF_LEN); + waitpid(pid, NULL, 0); + } + else{ + assert(false); + } + } + +} diff --git a/libsearch/index/inotify-index.h b/libsearch/index/inotify-index.h index 0ffa24b..e535b11 100644 --- a/libsearch/index/inotify-index.h +++ b/libsearch/index/inotify-index.h @@ -2,6 +2,7 @@ #define INOTIFYINDEX_H #include +#include #include #include #include "index-generator.h" @@ -9,6 +10,7 @@ #include "ukui-search-qdbus.h" #include "global-settings.h" #include "file-utils.h" +#include "first-index.h" #define BUF_LEN 1024 class InotifyIndex; @@ -29,6 +31,8 @@ public: bool AddWatch(const QString&); bool RemoveWatch(const QString&); virtual void DoSomething(const QFileInfo &) final; + + void eventProcess(const char*, ssize_t); protected: void run() override; private: diff --git a/src/main.cpp b/src/main.cpp index 0bb548e..7e56892 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include "qt-single-application.h" #include "qt-local-peer.h" //#include "inotify-manager.h" @@ -185,6 +186,7 @@ int main(int argc, char *argv[]) // FirstIndex* fi = new FirstIndex("/home/zhangzihao/Desktop/qwerty"); FirstIndex fi("/home"); fi.start(); + fi.wait(); // fi->wait(); // fi->exit(); // delete fi; From 501e252bf71d5de424a1cc34b16956cfdcc26417 Mon Sep 17 00:00:00 2001 From: zhangzihao Date: Thu, 21 Jan 2021 21:05:53 +0800 Subject: [PATCH 2/3] Adjust the logic between parent and child processes, named pipes to be added. --- libsearch/index/first-index.cpp | 32 +++++----------------- libsearch/index/inotify-index.cpp | 44 ++++++++++++++++++++++++------- libsearch/index/inotify-index.h | 2 +- libsearch/index/traverse_bfs.h | 1 + src/main.cpp | 2 +- 5 files changed, 43 insertions(+), 38 deletions(-) diff --git a/libsearch/index/first-index.cpp b/libsearch/index/first-index.cpp index 9ba13c9..120cffe 100644 --- a/libsearch/index/first-index.cpp +++ b/libsearch/index/first-index.cpp @@ -2,24 +2,6 @@ #include "first-index.h" #include -void handler(int){ - qDebug() << "Recieved SIGTERM!"; - GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2"); - GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2"); - GlobalSettings::getInstance()->setValue(INDEX_GENERATOR_NORMAL_EXIT, "2"); - GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "2"); - - - qDebug() << "indexDataBaseStatus: " << GlobalSettings::getInstance()->getValue(INDEX_DATABASE_STATE).toString(); - qDebug() << "contentIndexDataBaseStatus: " << GlobalSettings::getInstance()->getValue(CONTENT_INDEX_DATABASE_STATE).toString(); - -// InotifyIndex::getInstance("/home")->~InotifyIndex(); - qDebug() << "~IndexGenerator() end!" << endl; - - //wait linux kill this thread forcedly -// while (true); -} - #define NEW_QUEUE(a) a = new QQueue(); qDebug("---------------------------%s %s %s new at %d..",__FILE__,__FUNCTION__,#a,__LINE__); //#define DELETE_QUEUE(a ) @@ -94,11 +76,11 @@ void FirstIndex::run(){ else{ //if the parameter is false, index won't be rebuild //if it is true, index will be rebuild - this->p_indexGenerator = IndexGenerator::getInstance(true,this); + p_indexGenerator = IndexGenerator::getInstance(true,this); } } else{ - this->p_indexGenerator = IndexGenerator::getInstance(false,this); + p_indexGenerator = IndexGenerator::getInstance(false,this); } // this->q_content_index->enqueue(QString("/home/zhangzihao/Desktop/qwerty/四库全书.txt")); @@ -110,6 +92,7 @@ void FirstIndex::run(){ pid = fork(); if(pid == 0) { + prctl(PR_SET_PDEATHSIG, SIGKILL); prctl(PR_SET_NAME,"first-index"); FileUtils::_index_status = CREATING_INDEX; QSemaphore sem(5); @@ -166,9 +149,9 @@ void FirstIndex::run(){ if (this->q_content_index) delete this->q_content_index; this->q_content_index = nullptr; - if (this->p_indexGenerator) - delete this->p_indexGenerator; - this->p_indexGenerator = nullptr; + if (p_indexGenerator) + delete p_indexGenerator; + p_indexGenerator = nullptr; QThreadPool::globalInstance()->releaseThread(); QThreadPool::globalInstance()->waitForDone(); @@ -183,9 +166,6 @@ void FirstIndex::run(){ } FileUtils::_index_status = FINISH_CREATING_INDEX; - qDebug() << "sigset start!"; - sigset( SIGTERM, handler); - qDebug() << "sigset end!"; //quit() is shit!!! // return; diff --git a/libsearch/index/inotify-index.cpp b/libsearch/index/inotify-index.cpp index e87b75d..6ff5c22 100644 --- a/libsearch/index/inotify-index.cpp +++ b/libsearch/index/inotify-index.cpp @@ -1,5 +1,23 @@ #include "inotify-index.h" +void handler(int){ + qDebug() << "Recieved SIGTERM!"; + GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2"); + GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2"); + GlobalSettings::getInstance()->setValue(INDEX_GENERATOR_NORMAL_EXIT, "2"); + GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "2"); + + + qDebug() << "indexDataBaseStatus: " << GlobalSettings::getInstance()->getValue(INDEX_DATABASE_STATE).toString(); + qDebug() << "contentIndexDataBaseStatus: " << GlobalSettings::getInstance()->getValue(CONTENT_INDEX_DATABASE_STATE).toString(); + _exit(0); + +// InotifyIndex::getInstance("/home")->~InotifyIndex(); + + //wait linux kill this thread forcedly +// while (true); +} + InotifyIndex::InotifyIndex(const QString& path) : Traverse_BFS(path) { /*-------------ukuisearchdbus Test start-----------------*/ @@ -111,7 +129,7 @@ void InotifyIndex::eventProcess(const char* buf, ssize_t tmp){ if (event->mask & IN_CREATE){ if (event->mask & IN_ISDIR){ AddWatch(currentPath[event->wd] + '/' + event->name); - this->setPath(currentPath[event->wd] + '/' + event->name); + setPath(currentPath[event->wd] + '/' + event->name); Traverse(); } @@ -166,7 +184,7 @@ void InotifyIndex::eventProcess(const char* buf, ssize_t tmp){ RemoveWatch(currentPath[event->wd] + '/' + event->name); IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); AddWatch(currentPath[event->wd] + '/' + event->name); - this->setPath(currentPath[event->wd] + '/' + event->name); + setPath(currentPath[event->wd] + '/' + event->name); Traverse(); indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); @@ -266,6 +284,10 @@ next: */ void InotifyIndex::run(){ + qDebug() << "sigset start!"; + sigset( SIGTERM, handler); + qDebug() << "sigset end!"; + char buf[BUF_LEN] __attribute__((aligned(8))); ssize_t numRead; @@ -278,6 +300,7 @@ void InotifyIndex::run(){ pid = fork(); if(pid == 0) { + prctl(PR_SET_PDEATHSIG, SIGKILL); prctl(PR_SET_NAME,"inotify-index"); if (numRead == 0) { qDebug() << "read() from inotify fd returned 0!"; @@ -286,19 +309,20 @@ void InotifyIndex::run(){ qDebug() << "read"; } eventProcess(buf, numRead); - QTimer* liveTime = new QTimer(this); - bool b_timeout = false; - liveTime->setInterval(30000); - connect(liveTime, &QTimer::timeout, this, [&](){ -// _exit(0); - b_timeout = true; - }); + QTimer* liveTime = new QTimer(); + //restart inotify-index proccess per minute + liveTime->setInterval(60000); liveTime->start(); + +// connect(liveTime, &QTimer::timeout, [ = ](){ +//// _exit(0); +// *b_timeout = 1; +// }); for (;;){ numRead = read(m_fd, buf, BUF_LEN); liveTime->stop(); this->eventProcess(buf, numRead); - if (b_timeout){ + if (liveTime->remainingTime() < 1){ _exit(0); } liveTime->start(); diff --git a/libsearch/index/inotify-index.h b/libsearch/index/inotify-index.h index e535b11..56ef147 100644 --- a/libsearch/index/inotify-index.h +++ b/libsearch/index/inotify-index.h @@ -12,7 +12,7 @@ #include "file-utils.h" #include "first-index.h" -#define BUF_LEN 1024 +#define BUF_LEN 1024000 class InotifyIndex; static InotifyIndex* global_instance_of_index = nullptr; class InotifyIndex : public QThread, public Traverse_BFS diff --git a/libsearch/index/traverse_bfs.h b/libsearch/index/traverse_bfs.h index ed5e3c9..635eee6 100644 --- a/libsearch/index/traverse_bfs.h +++ b/libsearch/index/traverse_bfs.h @@ -10,6 +10,7 @@ class Traverse_BFS { public: void Traverse(); + virtual ~Traverse_BFS() = default; virtual void DoSomething(const QFileInfo&) = 0; void setPath(const QString&); protected: diff --git a/src/main.cpp b/src/main.cpp index 7e56892..f088c9b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -186,7 +186,7 @@ int main(int argc, char *argv[]) // FirstIndex* fi = new FirstIndex("/home/zhangzihao/Desktop/qwerty"); FirstIndex fi("/home"); fi.start(); - fi.wait(); +// fi.wait(); // fi->wait(); // fi->exit(); // delete fi; From add5b7bb464fecb697e2176bce9bda7cd1867a6c Mon Sep 17 00:00:00 2001 From: zhangzihao Date: Fri, 22 Jan 2021 09:49:44 +0800 Subject: [PATCH 3/3] Add named pipe to control the order of two child process. --- libsearch/file-utils.cpp | 2 ++ libsearch/file-utils.h | 8 ++++++++ libsearch/index/first-index.cpp | 22 +++++++++++++++++++++- libsearch/index/first-index.h | 7 ++++++- libsearch/index/inotify-index.cpp | 24 ++++++++++++++++++++++++ src/main.cpp | 13 ++++++++++++- 6 files changed, 73 insertions(+), 3 deletions(-) diff --git a/libsearch/file-utils.cpp b/libsearch/file-utils.cpp index 24a4c94..557cc52 100644 --- a/libsearch/file-utils.cpp +++ b/libsearch/file-utils.cpp @@ -11,6 +11,8 @@ #include #include #include "uchardet/uchardet.h" + + size_t FileUtils::_max_index_count = 0; size_t FileUtils::_current_index_count = 0; unsigned short FileUtils::_index_status = INITIAL_STATE; diff --git a/libsearch/file-utils.h b/libsearch/file-utils.h index 5594615..735b4c8 100644 --- a/libsearch/file-utils.h +++ b/libsearch/file-utils.h @@ -1,6 +1,12 @@ #ifndef FILEUTILS_H #define FILEUTILS_H #include "gobject-template.h" +#include +#include +#include +#include +#include +#include #include #include #include @@ -10,6 +16,8 @@ #define CREATING_INDEX 1 #define FINISH_CREATING_INDEX 2 +#define UKUI_SEARCH_PIPE_PATH "/tmp/ukuisearch" + class LIBSEARCH_EXPORT FileUtils { diff --git a/libsearch/index/first-index.cpp b/libsearch/index/first-index.cpp index 120cffe..5412d52 100644 --- a/libsearch/index/first-index.cpp +++ b/libsearch/index/first-index.cpp @@ -2,7 +2,7 @@ #include "first-index.h" #include -#define NEW_QUEUE(a) a = new QQueue(); qDebug("---------------------------%s %s %s new at %d..",__FILE__,__FUNCTION__,#a,__LINE__); +#define NEW_QUEUE(a) a = new QQueue(); qDebug("---------------------------%s %s %s new at %d..",__FILE__,__FUNCTION__,#a,__LINE__); //#define DELETE_QUEUE(a ) FirstIndex::FirstIndex(const QString& path) : Traverse_BFS(path) @@ -63,6 +63,18 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo){ } void FirstIndex::run(){ + int fifo_fd; + char buffer[2]; + memset(buffer, 0, sizeof(buffer)); + buffer[0] = 0x1; + buffer[1] = '\0'; + fifo_fd = open(UKUI_SEARCH_PIPE_PATH, O_RDWR); + if(fifo_fd == -1) + { + perror("open fifo error\n"); + assert(false); + } + if (this->bool_dataBaseExist){ if (this->bool_dataBaseStatusOK){ @@ -165,6 +177,14 @@ void FirstIndex::run(){ waitpid(pid,NULL,0); } + int retval = write(fifo_fd, buffer, strlen(buffer)); + if(retval == -1) + { + perror("write error\n"); + } + printf("write data ok!\n"); + close(fifo_fd); + FileUtils::_index_status = FINISH_CREATING_INDEX; //quit() is shit!!! diff --git a/libsearch/index/first-index.h b/libsearch/index/first-index.h index 794267d..6d45d98 100644 --- a/libsearch/index/first-index.h +++ b/libsearch/index/first-index.h @@ -6,7 +6,12 @@ #include #include #include -#include +#include +#include +#include +#include +#include +#include #include #include //#include diff --git a/libsearch/index/inotify-index.cpp b/libsearch/index/inotify-index.cpp index 6ff5c22..6e97091 100644 --- a/libsearch/index/inotify-index.cpp +++ b/libsearch/index/inotify-index.cpp @@ -284,6 +284,30 @@ next: */ void InotifyIndex::run(){ + int fifo_fd; + char buffer[2]; + memset(buffer, 0, sizeof(buffer)); + fifo_fd = open(UKUI_SEARCH_PIPE_PATH, O_RDWR); + if(fifo_fd == -1) + { + perror("open fifo error\n"); + assert(false); + } + int retval = read(fifo_fd, buffer, sizeof(buffer)); + if(retval == -1) + { + perror("read error\n"); + assert(false); + } + printf("read fifo=[%s]\n", buffer); + + printf("read data ok\n"); + close(fifo_fd); + if (buffer[0] & 0x1){ + printf("data confirmed\n"); + } + unlink(UKUI_SEARCH_PIPE_PATH); + qDebug() << "sigset start!"; sigset( SIGTERM, handler); qDebug() << "sigset end!"; diff --git a/src/main.cpp b/src/main.cpp index f088c9b..6878115 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include "qt-single-application.h" #include "qt-local-peer.h" //#include "inotify-manager.h" @@ -34,6 +33,7 @@ #include "global-settings.h" #include "xatom-helper.h" + void messageOutput(QtMsgType type, const QMessageLogContext &context, const QString &msg) { QByteArray localMsg = msg.toLocal8Bit(); @@ -92,6 +92,17 @@ void centerToScreen(QWidget* widget) { int main(int argc, char *argv[]) { + unlink(UKUI_SEARCH_PIPE_PATH); + int retval = mkfifo(UKUI_SEARCH_PIPE_PATH, 0777); + if(retval == -1) + { + perror("creat fifo error\n"); + assert(false); + return -1; + } + printf("create fifo success\n"); + + qInstallMessageHandler(messageOutput); qRegisterMetaType>("QPair"); qRegisterMetaType("Document");