Merge pull request #151 from MouseZhangZh/main

Reduced resource footprint and increased indexing time.

libsearch/global-settings.cpp

@@ -57,10 +57,6 @@ GlobalSettings::GlobalSettings(QObject *parent) : QObject(parent)
     }
 }
 
-GlobalSettings::~GlobalSettings()
-{
-}
-
 const QVariant GlobalSettings::getValue(const QString &key)
 {
     return m_cache.value(key);

libsearch/global-settings.h

@@ -74,7 +74,7 @@ public Q_SLOTS:
 
 private:
     explicit GlobalSettings(QObject *parent = nullptr);
-    ~GlobalSettings();
+    ~GlobalSettings() = default;
 
     QSettings* m_settings;
     QGSettings* m_gsettings;

libsearch/index/first-index.cpp

@@ -83,6 +83,7 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo){
 }
 
 void FirstIndex::run(){
+    QTime t1 = QTime::currentTime();
 
     int fifo_fd;
     char buffer[2];
@@ -120,7 +121,9 @@ void FirstIndex::run(){
             }
         }
         else{
-            p_indexGenerator = IndexGenerator::getInstance(false,this);
+//            p_indexGenerator = IndexGenerator::getInstance(false,this);
+            p_indexGenerator = IndexGenerator::getInstance(true,this);
+
         }
         QSemaphore sem(5);
         QMutex mutex1, mutex2, mutex3;
@@ -134,21 +137,38 @@ void FirstIndex::run(){
             this->setPath(QStandardPaths::writableLocation(QStandardPaths::HomeLocation));
             this->Traverse();
             FileUtils::_max_index_count = this->q_index->length();
+            qDebug()<<"max_index_count:"<<FileUtils::_max_index_count;
             sem.release(5);
         });
         QtConcurrent::run([&](){
             sem.acquire(2);
             mutex2.unlock();
             qDebug() << "index start;";
-            this->p_indexGenerator->creatAllIndex(this->q_index);
+            QQueue<QVector<QString>>* tmp = new QQueue<QVector<QString>>();
+            while (!this->q_index->empty()) {
+                for (size_t i = 0; (i < 8192) && (!this->q_index->empty()); ++i){
+                    tmp->enqueue(this->q_index->dequeue());
+                }
+                this->p_indexGenerator->creatAllIndex(tmp);
+                tmp->clear();
+            }
+            delete tmp;
             qDebug() << "index end;";
             sem.release(2);
         });
         QtConcurrent::run([&](){
             sem.acquire(2);
             mutex3.unlock();
-            qDebug() << "content index start;";
-            this->p_indexGenerator->creatAllIndex(this->q_content_index);
+            QQueue<QString>* tmp = new QQueue<QString>();;
+            while (!this->q_content_index->empty()) {
+//                for (size_t i = 0; (i < this->u_send_length) && (!this->q_content_index->empty()); ++i){
+                for (size_t i = 0; (i < 30) && (!this->q_content_index->empty()); ++i){
+                    tmp->enqueue(this->q_content_index->dequeue());
+                }
+                this->p_indexGenerator->creatAllIndex(tmp);
+                tmp->clear();
+            }
+            delete tmp;
             qDebug() << "content index end;";
             sem.release(2);
         });
@@ -190,6 +210,9 @@ void FirstIndex::run(){
         qWarning("write error\n");
     }
     qDebug("write data ok!\n");
+    QTime t2 = QTime::currentTime();
+    qWarning() << t1;
+    qWarning() << t2;
 
     return;
 

libsearch/index/first-index.h

@@ -70,6 +70,9 @@ private:
 //        QString(".xls"),
 //        QString(".xlsx"),
         QString(".txt")};
+
+    //xapian will auto commit per 10,000 changes, donnot change it!!!
+    const size_t u_send_length = 8192;
 };
 
 #endif // FIRSTINDEX_H

libsearch/index/index-generator.cpp

@@ -74,14 +74,14 @@ bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList)
 //        m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING);
 //        m_indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
 
-        int count =0;
+//        int count =0;
         for (auto i : *_doc_list_path){
 
             insertIntoDatabase(i);
-            if(++count > 8999){
-                count = 0;
-                m_database_path->commit();
-            }
+//            if(++count > 8999){
+//                count = 0;
+//                m_database_path->commit();
+//            }
         }
         m_database_path->commit();
     }
@@ -148,10 +148,10 @@ IndexGenerator::IndexGenerator(bool rebuild, QObject *parent) : QObject(parent)
     {
         QDir database(QString::fromStdString(INDEX_PATH));
         if(database.exists())
-            database.removeRecursively();
+            qDebug()<<"remove"<<database.removeRecursively();
         database.setPath(QString::fromStdString(CONTENT_INDEX_PATH));
         if(database.exists())
-            database.removeRecursively();
+            qDebug()<<"remove"<<database.removeRecursively();
     }
     m_database_path = new Xapian::WritableDatabase(INDEX_PATH, Xapian::DB_CREATE_OR_OPEN);
     m_database_content = new Xapian::WritableDatabase(CONTENT_INDEX_PATH, Xapian::DB_CREATE_OR_OPEN);
@@ -266,7 +266,8 @@ void IndexGenerator::HandlePathList(QQueue<QString> *messageList)
     ChineseSegmentation::getInstance();
     ConstructDocumentForContent *constructer;
     QThreadPool pool;
-    pool.setMaxThreadCount(((QThread::idealThreadCount() - 1) / 2) + 1);
+//    pool.setMaxThreadCount(((QThread::idealThreadCount() - 1) / 2) + 1);
+    pool.setMaxThreadCount(1);
     pool.setExpiryTimeout(100);
     while(!messageList->isEmpty())
     {

libsearch/index/inotify-index.cpp

@@ -402,7 +402,7 @@ fork:
             int rc;
             timeval* read_timeout = (timeval*)malloc(sizeof(timeval));
 
-            read_timeout->tv_sec = 60;
+            read_timeout->tv_sec = 40;
             read_timeout->tv_usec = 0;
             for(;;)
             {