From 5466590bec59d2acfe0f95f6267b5ca72559b247 Mon Sep 17 00:00:00 2001
From: MouseZhangZh <sendbypython@foxmail.com>
Date: Sat, 27 Feb 2021 10:51:19 +0800
Subject: [PATCH] Reduced resource footprint and increased indexing time.

---
 libsearch/global-settings.cpp       |  4 ----
 libsearch/global-settings.h         |  2 +-
 libsearch/index/first-index.cpp     | 31 +++++++++++++++++++++++++----
 libsearch/index/first-index.h       |  3 +++
 libsearch/index/index-generator.cpp | 17 ++++++++--------
 libsearch/index/inotify-index.cpp   |  2 +-
 6 files changed, 41 insertions(+), 18 deletions(-)

diff --git a/libsearch/global-settings.cpp b/libsearch/global-settings.cpp
index 4a84850..fdf3a35 100644
--- a/libsearch/global-settings.cpp
+++ b/libsearch/global-settings.cpp
@@ -57,10 +57,6 @@ GlobalSettings::GlobalSettings(QObject *parent) : QObject(parent)
     }
 }
 
-GlobalSettings::~GlobalSettings()
-{
-}
-
 const QVariant GlobalSettings::getValue(const QString &key)
 {
     return m_cache.value(key);
diff --git a/libsearch/global-settings.h b/libsearch/global-settings.h
index 0a8523e..7515b9d 100644
--- a/libsearch/global-settings.h
+++ b/libsearch/global-settings.h
@@ -74,7 +74,7 @@ public Q_SLOTS:
 
 private:
     explicit GlobalSettings(QObject *parent = nullptr);
-    ~GlobalSettings();
+    ~GlobalSettings() = default;
 
     QSettings* m_settings;
     QGSettings* m_gsettings;
diff --git a/libsearch/index/first-index.cpp b/libsearch/index/first-index.cpp
index e1cd3d3..c8ccee9 100644
--- a/libsearch/index/first-index.cpp
+++ b/libsearch/index/first-index.cpp
@@ -83,6 +83,7 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo){
 }
 
 void FirstIndex::run(){
+    QTime t1 = QTime::currentTime();
 
     int fifo_fd;
     char buffer[2];
@@ -120,7 +121,9 @@ void FirstIndex::run(){
             }
         }
         else{
-            p_indexGenerator = IndexGenerator::getInstance(false,this);
+//            p_indexGenerator = IndexGenerator::getInstance(false,this);
+            p_indexGenerator = IndexGenerator::getInstance(true,this);
+
         }
         QSemaphore sem(5);
         QMutex mutex1, mutex2, mutex3;
@@ -134,21 +137,38 @@ void FirstIndex::run(){
             this->setPath(QStandardPaths::writableLocation(QStandardPaths::HomeLocation));
             this->Traverse();
             FileUtils::_max_index_count = this->q_index->length();
+            qDebug()<<"max_index_count:"<<FileUtils::_max_index_count;
             sem.release(5);
         });
         QtConcurrent::run([&](){
             sem.acquire(2);
             mutex2.unlock();
             qDebug() << "index start;";
-            this->p_indexGenerator->creatAllIndex(this->q_index);
+            QQueue<QVector<QString>>* tmp = new QQueue<QVector<QString>>();
+            while (!this->q_index->empty()) {
+                for (size_t i = 0; (i < 8192) && (!this->q_index->empty()); ++i){
+                    tmp->enqueue(this->q_index->dequeue());
+                }
+                this->p_indexGenerator->creatAllIndex(tmp);
+                tmp->clear();
+            }
+            delete tmp;
             qDebug() << "index end;";
             sem.release(2);
         });
         QtConcurrent::run([&](){
             sem.acquire(2);
             mutex3.unlock();
-            qDebug() << "content index start;";
-            this->p_indexGenerator->creatAllIndex(this->q_content_index);
+            QQueue<QString>* tmp = new QQueue<QString>();;
+            while (!this->q_content_index->empty()) {
+//                for (size_t i = 0; (i < this->u_send_length) && (!this->q_content_index->empty()); ++i){
+                for (size_t i = 0; (i < 30) && (!this->q_content_index->empty()); ++i){
+                    tmp->enqueue(this->q_content_index->dequeue());
+                }
+                this->p_indexGenerator->creatAllIndex(tmp);
+                tmp->clear();
+            }
+            delete tmp;
             qDebug() << "content index end;";
             sem.release(2);
         });
@@ -190,6 +210,9 @@ void FirstIndex::run(){
         qWarning("write error\n");
     }
     qDebug("write data ok!\n");
+    QTime t2 = QTime::currentTime();
+    qWarning() << t1;
+    qWarning() << t2;
 
     return;
 
diff --git a/libsearch/index/first-index.h b/libsearch/index/first-index.h
index 1064efc..4641c50 100644
--- a/libsearch/index/first-index.h
+++ b/libsearch/index/first-index.h
@@ -70,6 +70,9 @@ private:
 //        QString(".xls"),
 //        QString(".xlsx"),
         QString(".txt")};
+
+    //xapian will auto commit per 10,000 changes, donnot change it!!!
+    const size_t u_send_length = 8192;
 };
 
 #endif // FIRSTINDEX_H
diff --git a/libsearch/index/index-generator.cpp b/libsearch/index/index-generator.cpp
index 3cc1a6d..333cd62 100644
--- a/libsearch/index/index-generator.cpp
+++ b/libsearch/index/index-generator.cpp
@@ -74,14 +74,14 @@ bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList)
 //        m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING);
 //        m_indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
 
-        int count =0;
+//        int count =0;
         for (auto i : *_doc_list_path){
 
             insertIntoDatabase(i);
-            if(++count > 8999){
-                count = 0;
-                m_database_path->commit();
-            }
+//            if(++count > 8999){
+//                count = 0;
+//                m_database_path->commit();
+//            }
         }
         m_database_path->commit();
     }
@@ -148,10 +148,10 @@ IndexGenerator::IndexGenerator(bool rebuild, QObject *parent) : QObject(parent)
     {
         QDir database(QString::fromStdString(INDEX_PATH));
         if(database.exists())
-            database.removeRecursively();
+            qDebug()<<"remove"<<database.removeRecursively();
         database.setPath(QString::fromStdString(CONTENT_INDEX_PATH));
         if(database.exists())
-            database.removeRecursively();
+            qDebug()<<"remove"<<database.removeRecursively();
     }
     m_database_path = new Xapian::WritableDatabase(INDEX_PATH, Xapian::DB_CREATE_OR_OPEN);
     m_database_content = new Xapian::WritableDatabase(CONTENT_INDEX_PATH, Xapian::DB_CREATE_OR_OPEN);
@@ -266,7 +266,8 @@ void IndexGenerator::HandlePathList(QQueue<QString> *messageList)
     ChineseSegmentation::getInstance();
     ConstructDocumentForContent *constructer;
     QThreadPool pool;
-    pool.setMaxThreadCount(((QThread::idealThreadCount() - 1) / 2) + 1);
+//    pool.setMaxThreadCount(((QThread::idealThreadCount() - 1) / 2) + 1);
+    pool.setMaxThreadCount(1);
     pool.setExpiryTimeout(100);
     while(!messageList->isEmpty())
     {
diff --git a/libsearch/index/inotify-index.cpp b/libsearch/index/inotify-index.cpp
index b3ee2ad..a224efc 100644
--- a/libsearch/index/inotify-index.cpp
+++ b/libsearch/index/inotify-index.cpp
@@ -402,7 +402,7 @@ fork:
             int rc;
             timeval* read_timeout = (timeval*)malloc(sizeof(timeval));
 
-            read_timeout->tv_sec = 60;
+            read_timeout->tv_sec = 40;
             read_timeout->tv_usec = 0;
             for(;;)
             {