Merge pull request #57 from iaom/0108-dev

Black list and search result queue.

libsearch/global-settings.cpp

@@ -16,6 +16,7 @@ GlobalSettings *GlobalSettings::getInstance()
 GlobalSettings::GlobalSettings(QObject *parent) : QObject(parent)
 {
     m_settings = new QSettings("org.ukui", "ukui-search", this);
+    m_block_dirs_settings = new QSettings("org.ukui","ukui-search-block-dirs",this);
     //the default number of transparency in mainwindow is 0.7
     //if someone changes the num in mainwindow, here should be modified too
     m_cache.insert(TRANSPARENCY_KEY, 0.7);
@@ -77,9 +78,27 @@ void GlobalSettings::resetAll()
     });
 }
 
-QList<QString> GlobalSettings::getBlockDirs()
+bool GlobalSettings::setBlockDirs(const QString &path, QString &returnMessage)
 {
-    return m_cache.keys();
+    QStringList blockDirs = m_block_dirs_settings->allKeys();
+    for(QString i:blockDirs)
+    {
+        if(path.startsWith(i))
+        {
+            returnMessage = QString(tr("Parent folder has been blocked!"));
+            return false;
+        }
+
+        if(i.startsWith(path))
+            m_block_dirs_settings->remove(i);
+    }
+    m_block_dirs_settings->setValue(path,"0");
+    return true;
+}
+
+QStringList GlobalSettings::getBlockDirs()
+{
+    return m_block_dirs_settings->allKeys();
 }
 
 void GlobalSettings::setValue(const QString &key, const QVariant &value)

libsearch/global-settings.h

@@ -34,7 +34,15 @@ public Q_SLOTS:
     void setValue(const QString&, const QVariant&);
     void reset(const QString&);
     void resetAll();
-    QList<QString> getBlockDirs();
+    /**
+     * @brief setBlockDirs
+     * set path for blacklist,return true if success,otherwise return false.
+     * @param path path to be blocked
+     * @param returnMessage this message will be set when return false.
+     * @return
+     */
+    bool setBlockDirs(const QString& path, QString &returnMessage);
+    QStringList getBlockDirs();
 
     void forceSync(const QString& = nullptr);
 
@@ -44,6 +52,7 @@ private:
 
     QSettings* m_settings;
     QGSettings* m_gsettings;
+    QSettings *m_block_dirs_settings;
     QMap<QString, QVariant> m_cache;
 
     QMutex m_mutex;

libsearch/index/file-searcher.cpp

@@ -1,82 +1,103 @@
-#include "file-searcher.h"
 #include <QFileInfo>
 #include <QDebug>
+#include <QtConcurrent>
 #include <chinese-segmentation.h>
+#include "file-searcher.h"
+#include "global-settings.h"
+
 
 FileSearcher::FileSearcher(QObject *parent) : QObject(parent)
 {
-
 }
 
-void FileSearcher::onKeywordSearch(QString keyword, int begin, int num)
+void FileSearcher::onKeywordSearch(QString keyword)
+{
+    //file
+    QtConcurrent::run([=](){
+        int begin = 0;
+        int num = 20;
+        int resultCount = 0;
+        while(1)
+        {
+            resultCount = keywordSearchfile(keyword,"1",1,begin,num);
+            if(resultCount == 0 || resultCount == -1)
+                break;
+            begin += num;
+        }
+    });
+    Q_EMIT this->resultFile(m_search_result_file);
+    //dir
+    QtConcurrent::run([=](){
+        int begin = 0;
+        int num = 20;
+        int resultCount = 0;
+        while(1)
+        {
+            resultCount = keywordSearchfile(keyword,"0",1,begin,num);
+            if(resultCount == 0 || resultCount == -1)
+                break;
+            begin += num;
+        }
+    });
+    Q_EMIT this->resultDir(m_search_result_dir);
+    //content
+    QtConcurrent::run([=](){
+        int begin = 0;
+        int num = 10;
+        int resultCount = 0;
+        while(1)
+        {
+            keywordSearchContent(keyword,begin,num);
+            if(resultCount == 0 || resultCount == -1)
+                break;
+            begin += num;
+        }
+    });
+    Q_EMIT this->resultContent(m_search_result_content);
+}
+
+int FileSearcher::keywordSearchfile(QString keyword, QString value, unsigned slot, int begin, int num)
 {
-    QVector<QStringList> searchResult;
     try
     {
         qDebug()<<"--search start--";
-
         Xapian::Database db(INDEX_PATH);
+        Xapian::Query query = creatQueryForFileSearch(keyword,db);
         Xapian::Enquire enquire(db);
-        Xapian::QueryParser qp;
-        qp.set_default_op(Xapian::Query::OP_PHRASE);
-        qp.set_database(db);
-        auto userInput = keyword;
 
-        std::string queryStr = keyword.replace(""," ").toStdString();
+        Xapian::Query queryFile;
-//        std::string s =db.get_spelling_suggestion(queryStr,10);
+        if(!value.isEmpty())
-//        qDebug()<<"spelling_suggestion!"<<QString::fromStdString(s);
-
-        qDebug()<<"queryStr!"<<QString::fromStdString(queryStr);
-        //Creat a query
-        Xapian::Query queryPhrase = qp.parse_query(queryStr,Xapian::QueryParser::FLAG_PHRASE);
-        std::vector<Xapian::Query> v;
-        for(int i=0;i<userInput.size();i++)
         {
-            v.push_back(Xapian::Query(QString(userInput.at(i)).toStdString()));
+            std::string slotValue = value.toStdString();
-            qDebug()<<userInput.at(i);
+            Xapian::Query queryValue = Xapian::Query(Xapian::Query::OP_VALUE_RANGE,slot,slotValue,slotValue);
-            qDebug()<<QString::fromStdString(Xapian::Query(QString(userInput.at(i)).toStdString()).get_description());
+            queryFile = Xapian::Query(Xapian::Query::OP_AND,query,queryValue);
+        }
+        else
+        {
+            queryFile = query;
         }
-        Xapian::Query queryNear =Xapian::Query(Xapian::Query::OP_NEAR, v.begin(), v.end());
-        Xapian::Query query = Xapian::Query(Xapian::Query::OP_AND,queryNear,queryPhrase);
 
-        //1- dir 2-file
+        qDebug()<<QString::fromStdString(queryFile.get_description());
-        unsigned slot = 1;
-        std::string value = "1";
-        Xapian::Query queryValue1 = Xapian::Query(Xapian::Query::OP_VALUE_GE,slot,value);
-        value = "0";
-        Xapian::Query queryValue0 = Xapian::Query(Xapian::Query::OP_VALUE_LE,1,value);
-        Xapian::Query queryDir = Xapian::Query(Xapian::Query::OP_AND,query,queryValue1);
-        Xapian::Query queryFile = Xapian::Query(Xapian::Query::OP_AND,query,queryValue0);
-
-        qDebug()<<QString::fromStdString(query.get_description());
-
-        enquire.set_query(queryDir);
-        //dir result
-        Xapian::MSet result = enquire.get_mset(begin, begin+num);
-        qDebug()<< "find dir results count=" <<static_cast<int>(result.get_matches_estimated());
-        searchResult.append(getResult(result));
 
         enquire.set_query(queryFile);
-        //file result
+        Xapian::MSet result = enquire.get_mset(begin, begin+num);
-        result = enquire.get_mset(begin, begin+num);
+        int resultCount =  static_cast<int>(result.get_matches_estimated());
-        qDebug()<< "find file results count=" <<static_cast<int>(result.get_matches_estimated());
+        qDebug()<< "find results count=" <<resultCount;
-        searchResult.append(getResult(result));
+        getResult(result,value);
 
         qDebug()<< "--search finish--";
+        return resultCount;
     }
     catch(const Xapian::Error &e)
     {
         qWarning() <<QString::fromStdString(e.get_description());
-        return;
+        qDebug()<< "--search finish--";
+        return -1;
     }
-    Q_EMIT this->result(searchResult);
-    return;
-
 }
 
-void FileSearcher::onKeywordSearchContent(QString keyword, int begin, int num)
+int FileSearcher::keywordSearchContent(QString keyword, int begin, int num)
 {
-    QMap<QString,QStringList> searchResult = QMap<QString,QStringList>();
     try
     {
         qDebug()<<"--content search start--";
@@ -108,24 +129,52 @@ void FileSearcher::onKeywordSearchContent(QString keyword, int begin, int num)
         enquire.set_query(query);
         //dir result
         Xapian::MSet result = enquire.get_mset(begin, begin+num);
-        qDebug()<< "find results count=" <<static_cast<int>(result.get_matches_estimated());
+        int resultCount = static_cast<int>(result.get_matches_estimated());
+        qDebug()<< "find results count=" <<resultCount;
 
-        searchResult = getContentResult(result,words);
+        getContentResult(result,words);
 
         qDebug()<< "--content search finish--";
+        return resultCount;
     }
     catch(const Xapian::Error &e)
     {
         qWarning() <<QString::fromStdString(e.get_description());
         qDebug()<< "--content search finish--";
-        return;
+        return -1;
     }
-    Q_EMIT this->contentResult(searchResult);
+}
-//    qDebug()<<searchResult;
+Xapian::Query FileSearcher::creatQueryForFileSearch(QString keyword, Xapian::Database &db)
-    return;
+{
+    Xapian::QueryParser qp;
+    qp.set_default_op(Xapian::Query::OP_PHRASE);
+    qp.set_database(db);
+    auto userInput = keyword;
+
+    std::string queryStr = keyword.replace(""," ").toStdString();
+//        std::string s =db.get_spelling_suggestion(queryStr,10);
+//        qDebug()<<"spelling_suggestion!"<<QString::fromStdString(s);
+
+    qDebug()<<"queryStr!"<<QString::fromStdString(queryStr);
+    //Creat a query
+    Xapian::Query queryPhrase = qp.parse_query(queryStr,Xapian::QueryParser::FLAG_PHRASE);
+    std::vector<Xapian::Query> v;
+    for(int i=0;i<userInput.size();i++)
+    {
+        v.push_back(Xapian::Query(QString(userInput.at(i)).toStdString()));
+        qDebug()<<QString::fromStdString(Xapian::Query(QString(userInput.at(i)).toStdString()).get_description());
+    }
+    Xapian::Query queryNear =Xapian::Query(Xapian::Query::OP_NEAR, v.begin(), v.end());
+
+    return Xapian::Query(Xapian::Query::OP_AND,queryNear,queryPhrase);
 }
 
-QStringList FileSearcher::getResult(Xapian::MSet &result)
+Xapian::Query FileSearcher::creatQueryForContentSearch(QString keyword, Xapian::Database &db)
+{
+
+}
+
+QStringList FileSearcher::getResult(Xapian::MSet &result, QString value)
 {
     //QStringList *pathTobeDelete = new QStringList;
     //Delete those path doc which is not already exist.
@@ -140,18 +189,34 @@ QStringList FileSearcher::getResult(Xapian::MSet &result)
         std::string data = doc.get_data();
         Xapian::weight docScoreWeight = it.get_weight();
         Xapian::percent docScorePercent = it.get_percent();
-        QFileInfo *info = new QFileInfo(QString::fromStdString(data));
+        QString path = QString::fromStdString(data);
+
+        if(isBlocked(path))
+            continue;
+
+        QFileInfo *info = new QFileInfo(path);
 
         if(!info->exists())
         {
 //                pathTobeDelete->append(QString::fromStdString(data));
-            qDebug()<<QString::fromStdString(data)<<"is not exist!!";
+            qDebug()<<path<<"is not exist!!";
         }
         else
         {
-            searchResult.append(QString::fromStdString(data));
+            switch (value.toInt())
+            {
+            case 1:
+                m_search_result_dir->enqueue(path);
+                break;
+            case 0:
+                m_search_result_file->enqueue(path);
+                break;
+            default:
+                break;
+            }
+            searchResult.append(path);
         }
-        qDebug()<< "doc="<< QString::fromStdString(data) << ",weight=" <<docScoreWeight << ",percent=" << docScorePercent;
+        qDebug()<< "doc="<< path << ",weight=" <<docScoreWeight << ",percent=" << docScorePercent;
     }
     //        if(!pathTobeDelete->isEmpty())
     //            deleteAllIndex(pathTobeDelete)
@@ -179,6 +244,10 @@ QMap<QString,QStringList> FileSearcher::getContentResult(Xapian::MSet &result, s
         double docScoreWeight = it.get_weight();
         Xapian::percent docScorePercent = it.get_percent();
         QString path = QString::fromStdString(doc.get_value(1));
+
+        if(isBlocked(path))
+            continue;
+
         QFileInfo *info = new QFileInfo(path);
 
         if(!info->exists())
@@ -200,6 +269,7 @@ QMap<QString,QStringList> FileSearcher::getContentResult(Xapian::MSet &result, s
             snippets.append(snippet);
             ++count;
         }
+        m_search_result_content->enqueue(qMakePair(path,snippets));
         searchResult.insert(path,snippets);
         qDebug()<< "path="<< path << ",weight=" <<docScoreWeight << ",percent=" << docScorePercent;
     }
@@ -207,3 +277,15 @@ QMap<QString,QStringList> FileSearcher::getContentResult(Xapian::MSet &result, s
     //            deleteAllIndex(pathTobeDelete)
     return searchResult;
 }
+
+bool FileSearcher::isBlocked(QString &path)
+{
+    QStringList blockList = GlobalSettings::getInstance()->getBlockDirs();
+    for(QString i :blockList)
+    {
+        if(path.startsWith(i))
+            return true;
+    }
+    return false;
+
+}

libsearch/index/file-searcher.h

@@ -6,6 +6,8 @@
 #include <QStandardPaths>
 #include <QVector>
 #include <QMap>
+#include <QQueue>
+#include <QPair>
 #define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/index_data").toStdString()
 #define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/content_index_data").toStdString()
 
@@ -17,15 +19,34 @@ public:
     explicit FileSearcher(QObject *parent = nullptr);
 
 public Q_SLOTS:
-    void onKeywordSearch(QString keyword, int begin = 0, int num = 20);
+    void onKeywordSearch(QString keyword);
-    void onKeywordSearchContent(QString keyword, int begin = 0, int num = 20);
 
 Q_SIGNALS:
-    void result(QVector<QStringList> resultP);
+    void resultFile(QQueue<QString> *);
-    void contentResult(QMap<QString,QStringList> resultC);
+    void resultDir(QQueue<QString> *);
+    void resultContent(QQueue<QPair<QString,QStringList>> *);
 private:
-    QStringList getResult(Xapian::MSet &result);
+    int keywordSearchfile(QString keyword, QString value,unsigned slot = 1,int begin = 0, int num = 20);
+    int keywordSearchContent(QString keyword, int begin = 0, int num = 20);
+
+    /**
+     * @brief FileSearcher::creatQueryForFileSearch
+     * This part shall be optimized frequently to provide a more stable search function.
+     * @param keyword
+     * @param db
+     * @return Xapian::Query
+     */
+    Xapian::Query creatQueryForFileSearch(QString keyword, Xapian::Database &db);
+    Xapian::Query creatQueryForContentSearch(QString keyword, Xapian::Database &db);
+
+    QStringList getResult(Xapian::MSet &result, QString value);
     QMap<QString,QStringList> getContentResult(Xapian::MSet &result,std::string &keyWord);
+
+    bool isBlocked(QString &path);
+    QQueue<QString> *m_search_result_file;
+    QQueue<QString> *m_search_result_dir;
+    QQueue<QPair<QString,QStringList>> *m_search_result_content;
+
 };
 
 #endif // FILESEARCHER_H

libsearch/index/index-generator.cpp

@@ -79,7 +79,7 @@ bool IndexGenerator::creatAllIndex(QList<QString> *messageList)
         {
             insertIntoContentDatabase(m_doc_list_content->at(i));
 
-            if(++count == 9999)
+            if(++count == 1000)
             {
                 count = 0;
                 m_database_content->commit();

src/mainwindow.cpp

@@ -255,25 +255,25 @@ void MainWindow::searchContent(QString searchcontent){
 
     //内容搜索测试用数据,每个文件（路径）对应一段文本内容
     FileSearcher *search = new FileSearcher();
-
+//iaom--------this part shall be rewrite
-    connect(search, &FileSearcher::contentResult, this, [ = ](QMap<QString,QStringList> map) {
+//    connect(search, &FileSearcher::contentResult, this, [ = ](QMap<QString,QStringList> map) {
-        m_types.append(SearchItem::SearchType::Contents);
+//        m_types.append(SearchItem::SearchType::Contents);
-        QStringList pathlist, contentList;
+//        QStringList pathlist, contentList;
-        qDebug() << map;
+//        qDebug() << map;
-        for (auto i : map.keys()){
+//        for (auto i : map.keys()){
-            QString temp;
+//            QString temp;
-            pathlist << i;
+//            pathlist << i;
-            for (auto s : map[i]){
+//            for (auto s : map[i]){
-                temp.append(s);
+//                temp.append(s);
-            }
+//            }
-            contentList.append(temp);
+//            contentList.append(temp);
-        }
+//        }
-        m_lists.append(pathlist);
+//        m_lists.append(pathlist);
-        m_contentFrame->setContentList(contentList);
+//        m_contentFrame->setContentList(contentList);
-    });
+//    });
     QTime t1 = QTime::currentTime();
 
-    search->onKeywordSearchContent(searchcontent);
+//    search->onKeywordSearch(searchcontent);
     QTime t2 = QTime::currentTime();
     qDebug() << t1;
     qDebug() << t2;
@@ -289,21 +289,21 @@ void MainWindow::searchContent(QString searchcontent){
     //文件搜索
 
     FileSearcher *searcher = new FileSearcher();
+//iaom--------this part shall be rewrite
+//    connect(searcher,&FileSearcher::result,[=](QVector<QStringList> resultV){
 
-    connect(searcher,&FileSearcher::result,[=](QVector<QStringList> resultV){
+//        QStringList list1 = resultV.at(0);
+//        QStringList list2 = resultV.at(1);
 
-        QStringList list1 = resultV.at(0);
+//        //        QVector<QStringList> lists;
-        QStringList list2 = resultV.at(1);
+//        m_lists.append(list1);
-
+//        m_lists.append(list2);
-        //        QVector<QStringList> lists;
+//        //        QVector<int> types;
-        m_lists.append(list1);
+//        m_types.append(SearchItem::SearchType::Dirs);
-        m_lists.append(list2);
+//        m_types.append(SearchItem::SearchType::Files);
-        //        QVector<int> types;
+//        m_contentFrame->refreshSearchList(m_types, m_lists, searchcontent);
-        m_types.append(SearchItem::SearchType::Dirs);
+//    });
-        m_types.append(SearchItem::SearchType::Files);
+//    searcher->onKeywordSearch(searchcontent,0,10);
-        m_contentFrame->refreshSearchList(m_types, m_lists, searchcontent);
-    });
-    searcher->onKeywordSearch(searchcontent,0,10);
     //    QStringList res = IndexGenerator::IndexSearch(searchcontent);
     //    types.append(SearchItem::SearchType::Files);
     //    lists.append(res);