openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Merge pull request #57 from iaom/0108-dev 

Black list and search result queue.
This commit is contained in:
张佳萍 2021-01-08 19:11:18 +08:00 committed by GitHub
parent 61c64186b6 f77e1c180b
commit 0e2cd05935
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 230 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
libsearch/global-settings.cpp
View File

@ -16,6 +16,7 @@ GlobalSettings *GlobalSettings::getInstance()
GlobalSettings::GlobalSettings(QObject *parent) : QObject(parent)
{
m_settings = new QSettings("org.ukui", "ukui-search", this);
m_block_dirs_settings = new QSettings("org.ukui","ukui-search-block-dirs",this);
//the default number of transparency in mainwindow is 0.7
//if someone changes the num in mainwindow, here should be modified too
m_cache.insert(TRANSPARENCY_KEY, 0.7);
@ -77,9 +78,27 @@ void GlobalSettings::resetAll()
});
}
QList<QString> GlobalSettings::getBlockDirs()
bool GlobalSettings::setBlockDirs(const QString &path, QString &returnMessage)
{
return m_cache.keys();
QStringList blockDirs = m_block_dirs_settings->allKeys();
for(QString i:blockDirs)
{
if(path.startsWith(i))
{
returnMessage = QString(tr("Parent folder has been blocked!"));
return false;
}
if(i.startsWith(path))
m_block_dirs_settings->remove(i);
}
m_block_dirs_settings->setValue(path,"0");
return true;
}
QStringList GlobalSettings::getBlockDirs()
{
return m_block_dirs_settings->allKeys();
}
void GlobalSettings::setValue(const QString &key, const QVariant &value)

11
libsearch/global-settings.h
View File

@ -34,7 +34,15 @@ public Q_SLOTS:
void setValue(const QString&, const QVariant&);
void reset(const QString&);
void resetAll();
QList<QString> getBlockDirs();
/**
* @brief setBlockDirs
* set path for blacklist,return true if success,otherwise return false.
* @param path path to be blocked
* @param returnMessage this message will be set when return false.
* @return
*/
bool setBlockDirs(const QString& path, QString &returnMessage);
QStringList getBlockDirs();
void forceSync(const QString& = nullptr);
@ -44,6 +52,7 @@ private:
QSettings* m_settings;
QGSettings* m_gsettings;
QSettings *m_block_dirs_settings;
QMap<QString, QVariant> m_cache;
QMutex m_mutex;

202
libsearch/index/file-searcher.cpp
View File

@ -1,82 +1,103 @@
#include "file-searcher.h"
#include <QFileInfo>
#include <QDebug>
#include <QtConcurrent>
#include <chinese-segmentation.h>
#include "file-searcher.h"
#include "global-settings.h"
FileSearcher::FileSearcher(QObject *parent) : QObject(parent)
{
}
void FileSearcher::onKeywordSearch(QString keyword, int begin, int num)
void FileSearcher::onKeywordSearch(QString keyword)
{
//file
QtConcurrent::run([=](){
int begin = 0;
int num = 20;
int resultCount = 0;
while(1)
{
resultCount = keywordSearchfile(keyword,"1",1,begin,num);
if(resultCount == 0 || resultCount == -1)
break;
begin += num;
}
});
Q_EMIT this->resultFile(m_search_result_file);
//dir
QtConcurrent::run([=](){
int begin = 0;
int num = 20;
int resultCount = 0;
while(1)
{
resultCount = keywordSearchfile(keyword,"0",1,begin,num);
if(resultCount == 0 || resultCount == -1)
break;
begin += num;
}
});
Q_EMIT this->resultDir(m_search_result_dir);
//content
QtConcurrent::run([=](){
int begin = 0;
int num = 10;
int resultCount = 0;
while(1)
{
keywordSearchContent(keyword,begin,num);
if(resultCount == 0 || resultCount == -1)
break;
begin += num;
}
});
Q_EMIT this->resultContent(m_search_result_content);
}
int FileSearcher::keywordSearchfile(QString keyword, QString value, unsigned slot, int begin, int num)
{
QVector<QStringList> searchResult;
try
{
qDebug()<<"--search start--";
Xapian::Database db(INDEX_PATH);
Xapian::Query query = creatQueryForFileSearch(keyword,db);
Xapian::Enquire enquire(db);
Xapian::QueryParser qp;
qp.set_default_op(Xapian::Query::OP_PHRASE);
qp.set_database(db);
auto userInput = keyword;
std::string queryStr = keyword.replace(""," ").toStdString();
// std::string s =db.get_spelling_suggestion(queryStr,10);
// qDebug()<<"spelling_suggestion!"<<QString::fromStdString(s);
qDebug()<<"queryStr!"<<QString::fromStdString(queryStr);
//Creat a query
Xapian::Query queryPhrase = qp.parse_query(queryStr,Xapian::QueryParser::FLAG_PHRASE);
std::vector<Xapian::Query> v;
for(int i=0;i<userInput.size();i++)
Xapian::Query queryFile;
if(!value.isEmpty())
{
v.push_back(Xapian::Query(QString(userInput.at(i)).toStdString()));
qDebug()<<userInput.at(i);
qDebug()<<QString::fromStdString(Xapian::Query(QString(userInput.at(i)).toStdString()).get_description());
std::string slotValue = value.toStdString();
Xapian::Query queryValue = Xapian::Query(Xapian::Query::OP_VALUE_RANGE,slot,slotValue,slotValue);
queryFile = Xapian::Query(Xapian::Query::OP_AND,query,queryValue);
}
else
{
queryFile = query;
}
Xapian::Query queryNear =Xapian::Query(Xapian::Query::OP_NEAR, v.begin(), v.end());
Xapian::Query query = Xapian::Query(Xapian::Query::OP_AND,queryNear,queryPhrase);
//1- dir 2-file
unsigned slot = 1;
std::string value = "1";
Xapian::Query queryValue1 = Xapian::Query(Xapian::Query::OP_VALUE_GE,slot,value);
value = "0";
Xapian::Query queryValue0 = Xapian::Query(Xapian::Query::OP_VALUE_LE,1,value);
Xapian::Query queryDir = Xapian::Query(Xapian::Query::OP_AND,query,queryValue1);
Xapian::Query queryFile = Xapian::Query(Xapian::Query::OP_AND,query,queryValue0);
qDebug()<<QString::fromStdString(query.get_description());
enquire.set_query(queryDir);
//dir result
Xapian::MSet result = enquire.get_mset(begin, begin+num);
qDebug()<< "find dir results count=" <<static_cast<int>(result.get_matches_estimated());
searchResult.append(getResult(result));
qDebug()<<QString::fromStdString(queryFile.get_description());
enquire.set_query(queryFile);
//file result
result = enquire.get_mset(begin, begin+num);
qDebug()<< "find file results count=" <<static_cast<int>(result.get_matches_estimated());
searchResult.append(getResult(result));
Xapian::MSet result = enquire.get_mset(begin, begin+num);
int resultCount = static_cast<int>(result.get_matches_estimated());
qDebug()<< "find results count=" <<resultCount;
getResult(result,value);
qDebug()<< "--search finish--";
return resultCount;
}
catch(const Xapian::Error &e)
{
qWarning() <<QString::fromStdString(e.get_description());
return;
qDebug()<< "--search finish--";
return -1;
}
Q_EMIT this->result(searchResult);
return;
}
void FileSearcher::onKeywordSearchContent(QString keyword, int begin, int num)
int FileSearcher::keywordSearchContent(QString keyword, int begin, int num)
{
QMap<QString,QStringList> searchResult = QMap<QString,QStringList>();
try
{
qDebug()<<"--content search start--";
@ -108,24 +129,52 @@ void FileSearcher::onKeywordSearchContent(QString keyword, int begin, int num)
enquire.set_query(query);
//dir result
Xapian::MSet result = enquire.get_mset(begin, begin+num);
qDebug()<< "find results count=" <<static_cast<int>(result.get_matches_estimated());
int resultCount = static_cast<int>(result.get_matches_estimated());
qDebug()<< "find results count=" <<resultCount;
searchResult = getContentResult(result,words);
getContentResult(result,words);
qDebug()<< "--content search finish--";
return resultCount;
}
catch(const Xapian::Error &e)
{
qWarning() <<QString::fromStdString(e.get_description());
qDebug()<< "--content search finish--";
return;
return -1;
}
Q_EMIT this->contentResult(searchResult);
// qDebug()<<searchResult;
return;
}
Xapian::Query FileSearcher::creatQueryForFileSearch(QString keyword, Xapian::Database &db)
{
Xapian::QueryParser qp;
qp.set_default_op(Xapian::Query::OP_PHRASE);
qp.set_database(db);
auto userInput = keyword;
std::string queryStr = keyword.replace(""," ").toStdString();
// std::string s =db.get_spelling_suggestion(queryStr,10);
// qDebug()<<"spelling_suggestion!"<<QString::fromStdString(s);
qDebug()<<"queryStr!"<<QString::fromStdString(queryStr);
//Creat a query
Xapian::Query queryPhrase = qp.parse_query(queryStr,Xapian::QueryParser::FLAG_PHRASE);
std::vector<Xapian::Query> v;
for(int i=0;i<userInput.size();i++)
{
v.push_back(Xapian::Query(QString(userInput.at(i)).toStdString()));
qDebug()<<QString::fromStdString(Xapian::Query(QString(userInput.at(i)).toStdString()).get_description());
}
Xapian::Query queryNear =Xapian::Query(Xapian::Query::OP_NEAR, v.begin(), v.end());
return Xapian::Query(Xapian::Query::OP_AND,queryNear,queryPhrase);
}
QStringList FileSearcher::getResult(Xapian::MSet &result)
Xapian::Query FileSearcher::creatQueryForContentSearch(QString keyword, Xapian::Database &db)
{
}
QStringList FileSearcher::getResult(Xapian::MSet &result, QString value)
{
//QStringList *pathTobeDelete = new QStringList;
//Delete those path doc which is not already exist.
@ -140,18 +189,34 @@ QStringList FileSearcher::getResult(Xapian::MSet &result)
std::string data = doc.get_data();
Xapian::weight docScoreWeight = it.get_weight();
Xapian::percent docScorePercent = it.get_percent();
QFileInfo *info = new QFileInfo(QString::fromStdString(data));
QString path = QString::fromStdString(data);
if(isBlocked(path))
continue;
QFileInfo *info = new QFileInfo(path);
if(!info->exists())
{
// pathTobeDelete->append(QString::fromStdString(data));
qDebug()<<QString::fromStdString(data)<<"is not exist!!";
qDebug()<<path<<"is not exist!!";
}
else
{
searchResult.append(QString::fromStdString(data));
switch (value.toInt())
{
case 1:
m_search_result_dir->enqueue(path);
break;
case 0:
m_search_result_file->enqueue(path);
break;
default:
break;
}
searchResult.append(path);
}
qDebug()<< "doc="<< QString::fromStdString(data) << ",weight=" <<docScoreWeight << ",percent=" << docScorePercent;
qDebug()<< "doc="<< path << ",weight=" <<docScoreWeight << ",percent=" << docScorePercent;
}
// if(!pathTobeDelete->isEmpty())
// deleteAllIndex(pathTobeDelete)
@ -179,6 +244,10 @@ QMap<QString,QStringList> FileSearcher::getContentResult(Xapian::MSet &result, s
double docScoreWeight = it.get_weight();
Xapian::percent docScorePercent = it.get_percent();
QString path = QString::fromStdString(doc.get_value(1));
if(isBlocked(path))
continue;
QFileInfo *info = new QFileInfo(path);
if(!info->exists())
@ -200,6 +269,7 @@ QMap<QString,QStringList> FileSearcher::getContentResult(Xapian::MSet &result, s
snippets.append(snippet);
++count;
}
m_search_result_content->enqueue(qMakePair(path,snippets));
searchResult.insert(path,snippets);
qDebug()<< "path="<< path << ",weight=" <<docScoreWeight << ",percent=" << docScorePercent;
}
@ -207,3 +277,15 @@ QMap<QString,QStringList> FileSearcher::getContentResult(Xapian::MSet &result, s
// deleteAllIndex(pathTobeDelete)
return searchResult;
}
bool FileSearcher::isBlocked(QString &path)
{
QStringList blockList = GlobalSettings::getInstance()->getBlockDirs();
for(QString i :blockList)
{
if(path.startsWith(i))
return true;
}
return false;
}

31
libsearch/index/file-searcher.h
View File

@ -6,6 +6,8 @@
#include <QStandardPaths>
#include <QVector>
#include <QMap>
#include <QQueue>
#include <QPair>
#define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/index_data").toStdString()
#define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/content_index_data").toStdString()
@ -17,15 +19,34 @@ public:
explicit FileSearcher(QObject *parent = nullptr);
public Q_SLOTS:
void onKeywordSearch(QString keyword, int begin = 0, int num = 20);
void onKeywordSearchContent(QString keyword, int begin = 0, int num = 20);
void onKeywordSearch(QString keyword);
Q_SIGNALS:
void result(QVector<QStringList> resultP);
void contentResult(QMap<QString,QStringList> resultC);
void resultFile(QQueue<QString> *);
void resultDir(QQueue<QString> *);
void resultContent(QQueue<QPair<QString,QStringList>> *);
private:
QStringList getResult(Xapian::MSet &result);
int keywordSearchfile(QString keyword, QString value,unsigned slot = 1,int begin = 0, int num = 20);
int keywordSearchContent(QString keyword, int begin = 0, int num = 20);
/**
* @brief FileSearcher::creatQueryForFileSearch
* This part shall be optimized frequently to provide a more stable search function.
* @param keyword
* @param db
* @return Xapian::Query
*/
Xapian::Query creatQueryForFileSearch(QString keyword, Xapian::Database &db);
Xapian::Query creatQueryForContentSearch(QString keyword, Xapian::Database &db);
QStringList getResult(Xapian::MSet &result, QString value);
QMap<QString,QStringList> getContentResult(Xapian::MSet &result,std::string &keyWord);
bool isBlocked(QString &path);
QQueue<QString> *m_search_result_file;
QQueue<QString> *m_search_result_dir;
QQueue<QPair<QString,QStringList>> *m_search_result_content;
};
#endif // FILESEARCHER_H

2
libsearch/index/index-generator.cpp
View File

@ -79,7 +79,7 @@ bool IndexGenerator::creatAllIndex(QList<QString> *messageList)
{
insertIntoContentDatabase(m_doc_list_content->at(i));
if(++count == 9999)
if(++count == 1000)
{
count = 0;
m_database_content->commit();

60
src/mainwindow.cpp
View File

@ -255,25 +255,25 @@ void MainWindow::searchContent(QString searchcontent){
//内容搜索测试用数据,每个文件（路径）对应一段文本内容
FileSearcher *search = new FileSearcher();
connect(search, &FileSearcher::contentResult, this, [ = ](QMap<QString,QStringList> map) {
m_types.append(SearchItem::SearchType::Contents);
QStringList pathlist, contentList;
qDebug() << map;
for (auto i : map.keys()){
QString temp;
pathlist << i;
for (auto s : map[i]){
temp.append(s);
}
contentList.append(temp);
}
m_lists.append(pathlist);
m_contentFrame->setContentList(contentList);
});
//iaom--------this part shall be rewrite
// connect(search, &FileSearcher::contentResult, this, [ = ](QMap<QString,QStringList> map) {
// m_types.append(SearchItem::SearchType::Contents);
// QStringList pathlist, contentList;
// qDebug() << map;
// for (auto i : map.keys()){
// QString temp;
// pathlist << i;
// for (auto s : map[i]){
// temp.append(s);
// }
// contentList.append(temp);
// }
// m_lists.append(pathlist);
// m_contentFrame->setContentList(contentList);
// });
QTime t1 = QTime::currentTime();
search->onKeywordSearchContent(searchcontent);
// search->onKeywordSearch(searchcontent);
QTime t2 = QTime::currentTime();
qDebug() << t1;
qDebug() << t2;
@ -289,21 +289,21 @@ void MainWindow::searchContent(QString searchcontent){
//文件搜索
FileSearcher *searcher = new FileSearcher();
//iaom--------this part shall be rewrite
// connect(searcher,&FileSearcher::result,[=](QVector<QStringList> resultV){
connect(searcher,&FileSearcher::result,[=](QVector<QStringList> resultV){
// QStringList list1 = resultV.at(0);
// QStringList list2 = resultV.at(1);
QStringList list1 = resultV.at(0);
QStringList list2 = resultV.at(1);
// QVector<QStringList> lists;
m_lists.append(list1);
m_lists.append(list2);
// QVector<int> types;
m_types.append(SearchItem::SearchType::Dirs);
m_types.append(SearchItem::SearchType::Files);
m_contentFrame->refreshSearchList(m_types, m_lists, searchcontent);
});
searcher->onKeywordSearch(searchcontent,0,10);
// // QVector<QStringList> lists;
// m_lists.append(list1);
// m_lists.append(list2);
// // QVector<int> types;
// m_types.append(SearchItem::SearchType::Dirs);
// m_types.append(SearchItem::SearchType::Files);
// m_contentFrame->refreshSearchList(m_types, m_lists, searchcontent);
// });
// searcher->onKeywordSearch(searchcontent,0,10);
// QStringList res = IndexGenerator::IndexSearch(searchcontent);
// types.append(SearchItem::SearchType::Files);
// lists.append(res);