Merge pull request #99 from MouseZhangZh/0121-dev-inzpf

🎉🎉🎉Use multiple processes to solve the problem of excessiving memory usage

debian/changelog

@@ -1,3 +1,9 @@
+ukui-search (0.0.1+0120) v101; urgency=medium
+
+  * Bugs Fixed.
+
+ -- zhangpengfei <zhangpengfei@kylinos.cn>  Wed, 20 Jan 2021 15:55:57 +0800
+
 ukui-search (0.0.1+0118) v101; urgency=medium
 
   * Feature: Add animation into inputbox.
@@ -11,8 +17,7 @@ ukui-search (0.0.1+0118) v101; urgency=medium
   * Fix: Translation is incompleted.(33047)
   * Fix: Widget crashed when searching & open file/filepath failed.
 
- -- zhangjiaping <zhangjiaping@zhangpengfei>  Mon, 18 Jan 2021 14:3
-6:12 +0800
+ -- zhangjiaping <zhangjiaping@kylinos.cn>  Mon, 18 Jan 2021 14:31:24 +0800
 
 ukui-search (0.0.1+0115) v101; urgency=medium
 
@@ -24,7 +29,7 @@ ukui-search (0.0.1+0114) v101; urgency=medium
 
   * Bugs fixed.
 
- -- zhangpengfei <zhangpengfei@zhangpengfei>  Thu, 14 Jan 2021 20:47:42 +0800
+ -- zhangpengfei <zhangpengfei@kylinos.cn>  Thu, 14 Jan 2021 20:47:42 +0800
 
 ukui-search (0.0.1+0113) v101; urgency=medium
 

libchinese-segmentation/chinese-segmentation.cpp

@@ -6,7 +6,6 @@ QMutex  ChineseSegmentation::m_mutex;
 
 ChineseSegmentation::ChineseSegmentation()
 {
-    QMutexLocker locker(&m_mutex);
     const char * const DICT_PATH = "/usr/share/ukui-search/res/dict/jieba.dict.utf8";
     const char * const  HMM_PATH = "/usr/share/ukui-search/res/dict/hmm_model.utf8";
     const char * const USER_DICT_PATH ="/usr/share/ukui-search/res/dict/user.dict.utf8";
@@ -29,6 +28,7 @@ ChineseSegmentation::~ChineseSegmentation()
 
 ChineseSegmentation *ChineseSegmentation::getInstance()
 {
+    QMutexLocker locker(&m_mutex);
     if (!global_instance_chinese_segmentation) {
         global_instance_chinese_segmentation = new ChineseSegmentation;
     }

libsearch/file-utils.cpp

@@ -11,6 +11,8 @@
 #include <QMimeType>
 #include <QQueue>
 #include "uchardet/uchardet.h"
+
+
 size_t FileUtils::_max_index_count = 0;
 size_t FileUtils::_current_index_count = 0;
 unsigned short FileUtils::_index_status = INITIAL_STATE;

libsearch/file-utils.h

@@ -1,6 +1,12 @@
 #ifndef FILEUTILS_H
 #define FILEUTILS_H
 #include "gobject-template.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 #include <QString>
 #include <QCryptographicHash>
 #include <QIcon>
@@ -10,6 +16,9 @@
 #define CREATING_INDEX 1
 #define FINISH_CREATING_INDEX 2
 
+#define UKUI_SEARCH_PIPE_PATH "/tmp/ukuisearch"
+
+
 class  LIBSEARCH_EXPORT FileUtils
 {
 public:

libsearch/index/file-searcher.cpp

@@ -66,6 +66,7 @@ void FileSearcher::onKeywordSearch(QString keyword,QQueue<QString> *searchResult
             total += resultCount;
             begin += num;
         }
+        return;
     });
 //    Q_EMIT this->resultFile(m_search_result_file);
     //dir
@@ -84,6 +85,7 @@ void FileSearcher::onKeywordSearch(QString keyword,QQueue<QString> *searchResult
             total += resultCount;
             begin += num;
         }
+        return;
     });
 //    Q_EMIT this->resultDir(m_search_result_dir);
     //content
@@ -103,6 +105,7 @@ void FileSearcher::onKeywordSearch(QString keyword,QQueue<QString> *searchResult
             total += resultCount;
             begin += num;
         }
+        return;
     });
 //    Q_EMIT this->resultContent(m_search_result_content);
 }

libsearch/index/first-index.cpp

@@ -2,25 +2,6 @@
 #include "first-index.h"
 #include <QDebug>
 
-
-void handler(int){
-    qDebug() << "Recieved SIGTERM!";
-    GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2");
-    GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2");
-    GlobalSettings::getInstance()->setValue(INDEX_GENERATOR_NORMAL_EXIT, "2");
-    GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "2");
-
-
-    qDebug() << "indexDataBaseStatus: " << GlobalSettings::getInstance()->getValue(INDEX_DATABASE_STATE).toString();
-    qDebug() << "contentIndexDataBaseStatus: " << GlobalSettings::getInstance()->getValue(CONTENT_INDEX_DATABASE_STATE).toString();
-
-//    InotifyIndex::getInstance("/home")->~InotifyIndex();
-    qDebug() << "~IndexGenerator() end!" << endl;
-
-    //wait linux kill this thread forcedly
-//    while (true);
-}
-
 #define NEW_QUEUE(a) a = new QQueue<QString>(); qDebug("---------------------------%s %s %s new at %d..",__FILE__,__FUNCTION__,#a,__LINE__);
 //#define DELETE_QUEUE(a )
 
@@ -82,6 +63,18 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo){
 }
 
 void FirstIndex::run(){
+    int fifo_fd;
+    char buffer[2];
+    memset(buffer, 0, sizeof(buffer));
+    buffer[0] = 0x1;
+    buffer[1] = '\0';
+    fifo_fd = open(UKUI_SEARCH_PIPE_PATH, O_RDWR);
+    if(fifo_fd == -1)
+    {
+        perror("open fifo error\n");
+        assert(false);
+    }
+
     if (this->bool_dataBaseExist){
         if (this->bool_dataBaseStatusOK){
 
@@ -95,11 +88,11 @@ void FirstIndex::run(){
         else{
             //if the parameter is false, index won't be rebuild
             //if it is true, index will be rebuild
-            this->p_indexGenerator = IndexGenerator::getInstance(true,this);
+            p_indexGenerator = IndexGenerator::getInstance(true,this);
         }
     }
     else{
-        this->p_indexGenerator = IndexGenerator::getInstance(false,this);
+        p_indexGenerator = IndexGenerator::getInstance(false,this);
     }
 
 //    this->q_content_index->enqueue(QString("/home/zhangzihao/Desktop/qwerty/四库全书.txt"));
@@ -107,6 +100,12 @@ void FirstIndex::run(){
 //    this->p_indexGenerator->creatAllIndex(this->q_content_index);
 
 
+    pid_t pid;
+    pid = fork();
+    if(pid  == 0)
+    {
+        prctl(PR_SET_PDEATHSIG, SIGKILL);
+        prctl(PR_SET_NAME,"first-index");
         FileUtils::_index_status = CREATING_INDEX;
         QSemaphore sem(5);
         QMutex mutex1, mutex2, mutex3;
@@ -144,6 +143,8 @@ void FirstIndex::run(){
         mutex1.unlock();
         mutex2.unlock();
         mutex3.unlock();
+        _exit(0);
+
 
 
 
@@ -151,7 +152,7 @@ void FirstIndex::run(){
         //don't use it now!!!!
         //MouseZhangZh
         //    this->~FirstIndex();
-//    qDebug() << "~FirstIndex end;";
+        //    qDebug() << "~FirstIndex end;"
 
 
         if (this->q_index)
@@ -160,19 +161,31 @@ void FirstIndex::run(){
         if (this->q_content_index)
             delete this->q_content_index;
         this->q_content_index = nullptr;
-    if (this->p_indexGenerator)
-        delete this->p_indexGenerator;
-    this->p_indexGenerator = nullptr;
+        if (p_indexGenerator)
+            delete p_indexGenerator;
+        p_indexGenerator = nullptr;
 
         QThreadPool::globalInstance()->releaseThread();
         QThreadPool::globalInstance()->waitForDone();
+    }
+    else if(pid < 0)
+    {
+        qWarning()<<"First Index fork error!!";
+    }
+    else
+    {
+        waitpid(pid,NULL,0);
+    }
 
-
+    int retval = write(fifo_fd, buffer, strlen(buffer));
+    if(retval == -1)
+    {
+        perror("write error\n");
+    }
+    printf("write data ok!\n");
+    close(fifo_fd);
 
     FileUtils::_index_status = FINISH_CREATING_INDEX;
-    qDebug() << "sigset start!";
-    sigset( SIGTERM, handler);
-    qDebug() << "sigset end!";
 
     //quit() is shit!!!
 //    return;

libsearch/index/first-index.h

@@ -4,6 +4,16 @@
 #include <QThread>
 #include <QtConcurrent/QtConcurrent>
 #include <signal.h>
+#include <QSemaphore>
+#include<sys/types.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+#include <sys/prctl.h>
 //#include <QtConcurrent>
 #include "traverse_bfs.h"
 #include "global-settings.h"

libsearch/index/index-generator.cpp

@@ -450,7 +450,11 @@ bool IndexGenerator::deleteAllIndex(QStringList *pathlist)
             qDebug()<<"delete path"<<doc;
             qDebug()<<"delete md5"<<QString::fromStdString(uniqueterm);
             m_database_path->commit();
+            m_database_content->commit();
             qDebug()<< "--delete finish--";
+//            qDebug()<<"m_database_path->get_lastdocid()!!!"<<m_database_path->get_lastdocid();
+
+//            qDebug()<<"m_database_path->get_doccount()!!!"<<m_database_path->get_doccount();
         }
         catch(const Xapian::Error &e)
         {

libsearch/index/inotify-index.cpp

@@ -1,5 +1,22 @@
 #include "inotify-index.h"
 
+void handler(int){
+    qDebug() << "Recieved SIGTERM!";
+    GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2");
+    GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2");
+    GlobalSettings::getInstance()->setValue(INDEX_GENERATOR_NORMAL_EXIT, "2");
+    GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "2");
+
+
+    qDebug() << "indexDataBaseStatus: " << GlobalSettings::getInstance()->getValue(INDEX_DATABASE_STATE).toString();
+    qDebug() << "contentIndexDataBaseStatus: " << GlobalSettings::getInstance()->getValue(CONTENT_INDEX_DATABASE_STATE).toString();
+    _exit(0);
+
+//    InotifyIndex::getInstance("/home")->~InotifyIndex();
+
+    //wait linux kill this thread forcedly
+//    while (true);
+}
 
 InotifyIndex::InotifyIndex(const QString& path) : Traverse_BFS(path)
 {
@@ -88,35 +105,20 @@ bool InotifyIndex::RemoveWatch(const QString &path){
     return true;
 }
 
-/*
- * Symbolic Link!!!!!!!!!!!!!!!!!!
- * Sysmbolic link to database dir will make a Infinite loop !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- * MouseZhangZh
-*/
-
-void InotifyIndex::run(){
-
-    char * p;
-    char buf[BUF_LEN] __attribute__((aligned(8)));
-
-    ssize_t numRead;
-
-    QQueue<QVector<QString>>* indexQueue = new QQueue<QVector<QString>>();
-    QQueue<QString>* contentIndexQueue = new QQueue<QString>();
-
-    for (;;) { /* Read events forever */
-        numRead = read(m_fd, buf, BUF_LEN);
-        if (numRead == 0) {
-            qDebug() << "read() from inotify fd returned 0!";
-        }
-        if (numRead == -1) {
-            qDebug() << "read";
-        }
+void InotifyIndex::eventProcess(const char* buf, ssize_t tmp){
     //        qDebug() << "Read " << numRead << " bytes from inotify fd";
 
     /* Process all of the events in buffer returned by read() */
 
-        for (p = buf; p < buf + numRead;) {
+
+    QQueue<QVector<QString>>* indexQueue = new QQueue<QVector<QString>>();
+    QQueue<QString>* contentIndexQueue = new QQueue<QString>();
+
+    ssize_t numRead = 0;
+    numRead = tmp;
+    char * p = const_cast<char*>(buf);
+
+    for (; p < buf + numRead;) {
         struct inotify_event * event = reinterpret_cast<inotify_event *>(p);
 //        qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
         if(event->name[0] != '.'){
@@ -127,7 +129,7 @@ void InotifyIndex::run(){
             if (event->mask & IN_CREATE){
                 if (event->mask & IN_ISDIR){
                     AddWatch(currentPath[event->wd] + '/' + event->name);
-                        this->setPath(currentPath[event->wd] + '/' + event->name);
+                    setPath(currentPath[event->wd] + '/' + event->name);
                     Traverse();
                 }
 
@@ -182,13 +184,9 @@ void InotifyIndex::run(){
                     RemoveWatch(currentPath[event->wd] + '/' + event->name);
                     IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
                     AddWatch(currentPath[event->wd] + '/' + event->name);
-                        this->setPath(currentPath[event->wd] + '/' + event->name);
+                    setPath(currentPath[event->wd] + '/' + event->name);
                     Traverse();
 
-                        //
-
-
-
                     indexQueue->enqueue(QVector<QString>() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0"));
                     IndexGenerator::getInstance()->creatAllIndex(indexQueue);
                     indexQueue->clear();
@@ -272,10 +270,95 @@ void InotifyIndex::run(){
 next:
         p += sizeof(struct inotify_event) + event->len;
     }
-    }
 
     delete indexQueue;
     indexQueue = nullptr;
     delete contentIndexQueue;
     contentIndexQueue = nullptr;
 }
+
+/*
+ * Symbolic Link!!!!!!!!!!!!!!!!!!
+ * Sysmbolic link to database dir will make a Infinite loop !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ * MouseZhangZh
+*/
+
+void InotifyIndex::run(){
+    int fifo_fd;
+    char buffer[2];
+    memset(buffer, 0, sizeof(buffer));
+    fifo_fd = open(UKUI_SEARCH_PIPE_PATH, O_RDWR);
+    if(fifo_fd == -1)
+    {
+        perror("open fifo error\n");
+        assert(false);
+    }
+    int retval = read(fifo_fd, buffer, sizeof(buffer));
+    if(retval == -1)
+    {
+        perror("read error\n");
+        assert(false);
+    }
+    printf("read fifo=[%s]\n", buffer);
+
+    printf("read data ok\n");
+    close(fifo_fd);
+    if (buffer[0] & 0x1){
+        printf("data confirmed\n");
+    }
+    unlink(UKUI_SEARCH_PIPE_PATH);
+
+    qDebug() << "sigset start!";
+    sigset( SIGTERM, handler);
+    qDebug() << "sigset end!";
+
+    char buf[BUF_LEN] __attribute__((aligned(8)));
+
+    ssize_t numRead;
+
+    for (;;) { /* Read events forever */
+        numRead = read(m_fd, buf, BUF_LEN);
+
+
+        pid_t pid;
+        pid = fork();
+        if(pid  == 0)
+        {
+            prctl(PR_SET_PDEATHSIG, SIGKILL);
+            prctl(PR_SET_NAME,"inotify-index");
+            if (numRead == 0) {
+                qDebug() << "read() from inotify fd returned 0!";
+            }
+            if (numRead == -1) {
+                qDebug() << "read";
+            }
+            eventProcess(buf, numRead);
+            QTimer* liveTime = new QTimer();
+            //restart inotify-index proccess per minute
+            liveTime->setInterval(60000);
+            liveTime->start();
+
+//            connect(liveTime, &QTimer::timeout, [ = ](){
+////                _exit(0);
+//                *b_timeout = 1;
+//            });
+            for (;;){
+                numRead = read(m_fd, buf, BUF_LEN);
+                liveTime->stop();
+                this->eventProcess(buf, numRead);
+                if (liveTime->remainingTime() < 1){
+                    _exit(0);
+                }
+                liveTime->start();
+            }
+        }
+        else if (pid > 0){
+            memset(buf, 0x00, BUF_LEN);
+            waitpid(pid, NULL, 0);
+        }
+        else{
+            assert(false);
+        }
+    }
+
+}

libsearch/index/inotify-index.h

@@ -2,6 +2,7 @@
 #define INOTIFYINDEX_H
 
 #include <QThread>
+#include <QTimer>
 #include <unistd.h>
 #include <sys/inotify.h>
 #include "index-generator.h"
@@ -9,8 +10,9 @@
 #include "ukui-search-qdbus.h"
 #include "global-settings.h"
 #include "file-utils.h"
+#include "first-index.h"
 
-#define BUF_LEN 1024
+#define BUF_LEN 1024000
 class InotifyIndex;
 static InotifyIndex* global_instance_of_index = nullptr;
 class InotifyIndex : public QThread, public Traverse_BFS
@@ -29,6 +31,8 @@ public:
     bool AddWatch(const QString&);
     bool RemoveWatch(const QString&);
     virtual void DoSomething(const QFileInfo &) final;
+
+    void eventProcess(const char*, ssize_t);
 protected:
     void run() override;
 private:

libsearch/index/traverse_bfs.h

@@ -10,6 +10,7 @@ class Traverse_BFS
 {
 public:
     void Traverse();
+    virtual ~Traverse_BFS() = default;
     virtual void DoSomething(const QFileInfo&) = 0;
     void setPath(const QString&);
 protected:

src/main.cpp

@@ -33,6 +33,7 @@
 #include "global-settings.h"
 #include "xatom-helper.h"
 
+
 void messageOutput(QtMsgType type, const QMessageLogContext &context, const QString &msg)
 {
     QByteArray localMsg = msg.toLocal8Bit();
@@ -91,6 +92,17 @@ void centerToScreen(QWidget* widget) {
 
 int main(int argc, char *argv[])
 {
+    unlink(UKUI_SEARCH_PIPE_PATH);
+    int retval = mkfifo(UKUI_SEARCH_PIPE_PATH, 0777);
+    if(retval == -1)
+    {
+        perror("creat fifo error\n");
+        assert(false);
+        return -1;
+    }
+    printf("create fifo success\n");
+
+
     qInstallMessageHandler(messageOutput);
     qRegisterMetaType<QPair<QString,QStringList>>("QPair<QString,QStringList>");
     qRegisterMetaType<Document>("Document");
@@ -185,6 +197,7 @@ int main(int argc, char *argv[])
 //    FirstIndex* fi = new FirstIndex("/home/zhangzihao/Desktop/qwerty");
     FirstIndex fi("/home");
     fi.start();
+//    fi.wait();
 //    fi->wait();
 //    fi->exit();
 //    delete fi;