p90463782
/
ukui-search
forked from openkylin/ukui-search
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #62 from iaom/0108-dev 

修改搜索和索引接口，增加取消黑名单功能。
This commit is contained in:
Mouse Zhang 2021-01-10 16:24:07 +08:00 committed by GitHub
parent af19c6c94d 929063645b
commit 2b89f0a414
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 221 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libsearch/file-utils.cpp
View File

@ -10,6 +10,8 @@
#include <QMimeDatabase>
#include <QMimeType>
#include <QQueue>
size_t FileUtils::_max_index_count = 0;
size_t FileUtils::_current_index_count = 0;
QMap<QString, QStringList> FileUtils::map_chinese2pinyin = QMap<QString, QStringList>();
FileUtils::FileUtils()

2
libsearch/file-utils.h
View File

@ -29,6 +29,8 @@ public:
static QString getMimetype(QString &path, bool getsuffix = false);
static QString *getDocxTextContent(QString &path);
static QString *getTxtContent(QString &path);
static size_t _max_index_count;
static size_t _current_index_count;
private:
FileUtils();

16
libsearch/global-settings.cpp
View File

@ -80,22 +80,28 @@ void GlobalSettings::resetAll()
});
}
bool GlobalSettings::setBlockDirs(const QString &path, QString &returnMessage)
bool GlobalSettings::setBlockDirs(const QString &path, QString &returnMessage, bool remove)
{
//why QSetting's key can't start with "/"??
QString pathKey = path.right(path.length()-1);
if(remove)
{
m_block_dirs_settings->remove(pathKey);
return true;
}
QStringList blockDirs = m_block_dirs_settings->allKeys();
for(QString i:blockDirs)
{
// qWarning()<<i;
if(path.right(path.length()-1).startsWith(i))
if(pathKey.startsWith(i))
{
returnMessage = QString(tr("Parent folder has been blocked!"));
return false;
}
if(i.startsWith(path.right(path.length()-1)))
if(i.startsWith(pathKey))
m_block_dirs_settings->remove(i);
}
m_block_dirs_settings->setValue(path.right(path.length()-1),"0");
m_block_dirs_settings->setValue(pathKey,"0");
return true;
}

3
libsearch/global-settings.h
View File

@ -39,9 +39,10 @@ public Q_SLOTS:
* set path for blacklist,return true if success,otherwise return false.
* @param path path to be blocked
* @param returnMessage this message will be set when return false.
* @param true to remove blocking,false to set blocking,default set false.
* @return
*/
bool setBlockDirs(const QString& path, QString &returnMessage);
bool setBlockDirs(const QString& path, QString &returnMessage,bool remove = false);
QStringList getBlockDirs();
void forceSync(const QString& = nullptr);

183
libsearch/index/file-searcher.cpp
View File

@ -1,65 +1,97 @@
#include <QFileInfo>
#include <QDebug>
#include <QtConcurrent>
#include <QThread>
#include <chinese-segmentation.h>
#include "file-searcher.h"
#include "global-settings.h"
size_t FileSearcher::uniqueSymbol1 = 0;
size_t FileSearcher::uniqueSymbol2 = 0;
size_t FileSearcher::uniqueSymbol3 = 0;
QMutex FileSearcher::m_mutex1;
QMutex FileSearcher::m_mutex2;
QMutex FileSearcher::m_mutex3;
FileSearcher::FileSearcher(QObject *parent) : QObject(parent)
{
}
void FileSearcher::onKeywordSearch(QString keyword)
FileSearcher::~FileSearcher()
{
m_search_result_file = new QQueue<QString>;
m_search_result_dir = new QQueue<QString>;
m_search_result_content = new QQueue<QPair<QString,QStringList>>;
//file
QtConcurrent::run([=](){
int begin = 0;
int num = 20;
int resultCount = 0;
while(1)
{
resultCount = keywordSearchfile(keyword,"1",1,begin,num);
if(resultCount == 0 || resultCount == -1)
break;
begin += num;
}
});
Q_EMIT this->resultFile(m_search_result_file);
//dir
QtConcurrent::run([=](){
int begin = 0;
int num = 20;
int resultCount = 0;
while(1)
{
resultCount = keywordSearchfile(keyword,"0",1,begin,num);
if(resultCount == 0 || resultCount == -1)
break;
begin += num;
}
});
Q_EMIT this->resultDir(m_search_result_dir);
//content
QtConcurrent::run([=](){
int begin = 0;
int num = 10;
int resultCount = 0;
while(1)
{
keywordSearchContent(keyword,begin,num);
if(resultCount == 0 || resultCount == -1)
break;
begin += num;
}
});
Q_EMIT this->resultContent(m_search_result_content);
}
int FileSearcher::keywordSearchfile(QString keyword, QString value, unsigned slot, int begin, int num)
void FileSearcher::onKeywordSearch(QString keyword,QQueue<QString> *searchResultFile,QQueue<QString> *searchResultDir,QQueue<QPair<QString,QStringList>> *searchResultContent)
{
m_search_result_file = searchResultFile;
m_search_result_dir = searchResultDir;
m_search_result_content = searchResultContent;
m_mutex1.lock();
++uniqueSymbol1;
m_mutex1.unlock();
m_mutex2.lock();
++uniqueSymbol2;
m_mutex2.unlock();
m_mutex3.lock();
++uniqueSymbol3;
m_mutex3.unlock();
//file
QtConcurrent::run([&, uniqueSymbol1](){
if(!m_search_result_file->isEmpty())
m_search_result_file->clear();
int begin = 0;
int num = 5;
int resultCount = 0;
int total = 0;
while(total<20)
{
resultCount = keywordSearchfile(uniqueSymbol1,keyword,"0",1,begin,num);
if(resultCount == 0 || resultCount == -1)
break;
total += resultCount;
begin += num;
}
});
// Q_EMIT this->resultFile(m_search_result_file);
//dir
QtConcurrent::run([&, uniqueSymbol2](){
if(!m_search_result_dir->isEmpty())
m_search_result_dir->clear();
int begin = 0;
int num = 5;
int resultCount = 0;
int total = 0;
while(total<20)
{
resultCount = keywordSearchfile(uniqueSymbol2,keyword,"1",1,begin,num);
if(resultCount == 0 || resultCount == -1)
break;
total += resultCount;
begin += num;
}
});
// Q_EMIT this->resultDir(m_search_result_dir);
//content
QtConcurrent::run([&, uniqueSymbol3](){
if(!m_search_result_content->isEmpty())
m_search_result_content->clear();
int begin = 0;
int num = 5;
int resultCount = 0;
int total = 0;
while(total<20)
{
keywordSearchContent(uniqueSymbol3,keyword,begin,num);
if(resultCount == 0 || resultCount == -1)
break;
total += resultCount;
begin += num;
}
});
// Q_EMIT this->resultContent(m_search_result_content);
}
int FileSearcher::keywordSearchfile(size_t uniqueSymbol, QString keyword, QString value, unsigned slot, int begin, int num)
{
try
{
@ -86,7 +118,7 @@ int FileSearcher::keywordSearchfile(QString keyword, QString value, unsigned slo
Xapian::MSet result = enquire.get_mset(begin, begin+num);
int resultCount = static_cast<int>(result.get_matches_estimated());
qDebug()<< "find results count=" <<resultCount;
getResult(result,value);
getResult(uniqueSymbol, result, value);
qDebug()<< "--search finish--";
return resultCount;
@ -99,7 +131,7 @@ int FileSearcher::keywordSearchfile(QString keyword, QString value, unsigned slo
}
}
int FileSearcher::keywordSearchContent(QString keyword, int begin, int num)
int FileSearcher::keywordSearchContent(size_t uniqueSymbol, QString keyword, int begin, int num)
{
try
{
@ -135,7 +167,7 @@ int FileSearcher::keywordSearchContent(QString keyword, int begin, int num)
int resultCount = static_cast<int>(result.get_matches_estimated());
qDebug()<< "find results count=" <<resultCount;
getContentResult(result,words);
getContentResult(uniqueSymbol, result, words);
qDebug()<< "--content search finish--";
return resultCount;
@ -177,7 +209,7 @@ Xapian::Query FileSearcher::creatQueryForContentSearch(QString keyword, Xapian::
}
QStringList FileSearcher::getResult(Xapian::MSet &result, QString value)
QStringList FileSearcher::getResult(size_t uniqueSymbol, Xapian::MSet &result, QString value)
{
//QStringList *pathTobeDelete = new QStringList;
//Delete those path doc which is not already exist.
@ -197,9 +229,9 @@ QStringList FileSearcher::getResult(Xapian::MSet &result, QString value)
if(isBlocked(path))
continue;
QFileInfo *info = new QFileInfo(path);
QFileInfo info(path);
if(!info->exists())
if(!info.exists())
{
// pathTobeDelete->append(QString::fromStdString(data));
qDebug()<<path<<"is not exist!!";
@ -209,10 +241,31 @@ QStringList FileSearcher::getResult(Xapian::MSet &result, QString value)
switch (value.toInt())
{
case 1:
m_search_result_dir->enqueue(path);
m_mutex1.lock();
if(uniqueSymbol == FileSearcher::uniqueSymbol1)
{
m_search_result_dir->enqueue(path);
m_mutex1.unlock();
}
else
{
m_mutex1.unlock();
exit(0);
}
break;
case 0:
m_search_result_file->enqueue(path);
m_mutex2.lock();
if(uniqueSymbol == FileSearcher::uniqueSymbol2)
{
m_search_result_file->enqueue(path);
m_mutex2.unlock();
}
else
{
m_mutex2.unlock();
exit(0);
}
break;
default:
break;
@ -226,7 +279,7 @@ QStringList FileSearcher::getResult(Xapian::MSet &result, QString value)
return searchResult;
}
QMap<QString,QStringList> FileSearcher::getContentResult(Xapian::MSet &result, std::string &keyWord)
QMap<QString,QStringList> FileSearcher::getContentResult(size_t uniqueSymbol, Xapian::MSet &result, std::string &keyWord)
{
//QStringList *pathTobeDelete = new QStringList;
//Delete those path doc which is not already exist.
@ -251,9 +304,9 @@ QMap<QString,QStringList> FileSearcher::getContentResult(Xapian::MSet &result, s
if(isBlocked(path))
continue;
QFileInfo *info = new QFileInfo(path);
QFileInfo info(path);
if(!info->exists())
if(!info.exists())
{
// pathTobeDelete->append(QString::fromStdString(data));
qDebug()<<path<<"is not exist!!";
@ -272,7 +325,17 @@ QMap<QString,QStringList> FileSearcher::getContentResult(Xapian::MSet &result, s
snippets.append(snippet);
++count;
}
m_search_result_content->enqueue(qMakePair(path,snippets));
m_mutex3.lock();
if(uniqueSymbol == FileSearcher::uniqueSymbol3)
{
m_search_result_content->enqueue(qMakePair(path,snippets));
m_mutex3.unlock();
}
else
{
m_mutex3.unlock();
exit(0);
}
searchResult.insert(path,snippets);
qDebug()<< "path="<< path << ",weight=" <<docScoreWeight << ",percent=" << docScorePercent;
}

21
libsearch/index/file-searcher.h
View File

@ -8,6 +8,7 @@
#include <QMap>
#include <QQueue>
#include <QPair>
#include <QMutex>
#define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/index_data").toStdString()
#define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/content_index_data").toStdString()
@ -17,17 +18,18 @@ class FileSearcher : public QObject
Q_OBJECT
public:
explicit FileSearcher(QObject *parent = nullptr);
~FileSearcher();
public Q_SLOTS:
void onKeywordSearch(QString keyword);
void onKeywordSearch(QString keyword,QQueue<QString> *searchResultFile,QQueue<QString> *searchResultDir,QQueue<QPair<QString,QStringList>> *searchResultContent);
Q_SIGNALS:
void resultFile(QQueue<QString> *);
void resultDir(QQueue<QString> *);
void resultContent(QQueue<QPair<QString,QStringList>> *);
private:
int keywordSearchfile(QString keyword, QString value,unsigned slot = 1,int begin = 0, int num = 20);
int keywordSearchContent(QString keyword, int begin = 0, int num = 20);
int keywordSearchfile(size_t uniqueSymbol, QString keyword, QString value,unsigned slot = 1,int begin = 0, int num = 20);
int keywordSearchContent(size_t uniqueSymbol, QString keyword, int begin = 0, int num = 20);
/**
* @brief FileSearcher::creatQueryForFileSearch
@ -39,14 +41,21 @@ private:
Xapian::Query creatQueryForFileSearch(QString keyword, Xapian::Database &db);
Xapian::Query creatQueryForContentSearch(QString keyword, Xapian::Database &db);
QStringList getResult(Xapian::MSet &result, QString value);
QMap<QString,QStringList> getContentResult(Xapian::MSet &result,std::string &keyWord);
QStringList getResult(size_t uniqueSymbol, Xapian::MSet &result, QString value);
QMap<QString,QStringList> getContentResult(size_t uniqueSymbol, Xapian::MSet &result,std::string &keyWord);
bool isBlocked(QString &path);
QQueue<QString> *m_search_result_file = nullptr;
QQueue<QString> *m_search_result_dir = nullptr;
QQueue<QPair<QString,QStringList>> *m_search_result_content = nullptr;
bool m_searching = false;
static size_t uniqueSymbol1;
static size_t uniqueSymbol2;
static size_t uniqueSymbol3;
static QMutex m_mutex1;
static QMutex m_mutex2;
static QMutex m_mutex3;
};
#endif // FILESEARCHER_H

2
libsearch/index/filetypefilter.cpp
View File

@ -38,7 +38,7 @@ void FileTypeFilter::Test(){
// this->result = new QList<QString>();
// this->result->append(QString("/home/zpf/桌面/DOCX 文档(1).docx"));
ig->creatAllIndex(this->result);
// ig->creatAllIndex(this->result);
}

10
libsearch/index/index-generator.cpp
View File

@ -17,9 +17,11 @@ using namespace std;
#define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/content_index_data").toStdString()
static IndexGenerator *global_instance = nullptr;
QMutex IndexGenerator::m_mutex;
IndexGenerator *IndexGenerator::getInstance(bool rebuild)
{
QMutexLocker locker(&m_mutex);
if (!global_instance) {
global_instance = new IndexGenerator(rebuild);
}
@ -32,7 +34,7 @@ bool IndexGenerator::setIndexdataPath()
}
//文件名索引
bool IndexGenerator::creatAllIndex(QList<QVector<QString> > *messageList)
bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList)
{
HandlePathList(messageList);
try
@ -69,7 +71,7 @@ bool IndexGenerator::creatAllIndex(QList<QVector<QString> > *messageList)
return true;
}
//文件内容索引
bool IndexGenerator::creatAllIndex(QList<QString> *messageList)
bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList)
{
HandlePathList(messageList);
try
@ -152,7 +154,7 @@ void IndexGenerator::insertIntoContentDatabase(Document doc)
return;
}
void IndexGenerator::HandlePathList(QList<QVector<QString>> *messageList)
void IndexGenerator::HandlePathList(QQueue<QVector<QString>> *messageList)
{
qDebug()<<"Begin HandlePathList!";
qDebug()<<messageList->size();
@ -169,7 +171,7 @@ void IndexGenerator::HandlePathList(QList<QVector<QString>> *messageList)
return;
}
void IndexGenerator::HandlePathList(QList<QString> *messageList)
void IndexGenerator::HandlePathList(QQueue<QString> *messageList)
{
qDebug()<<"Begin HandlePathList for content index!";
qDebug()<<messageList->size();

11
libsearch/index/index-generator.h
View File

@ -7,6 +7,8 @@
#include <QStringList>
#include <QMap>
#include <QCryptographicHash>
#include <QMutex>
#include <QQueue>
#include "document.h"
#include "file-reader.h"
@ -24,16 +26,17 @@ Q_SIGNALS:
void transactionFinished();
void searchFinish();
public Q_SLOTS:
bool creatAllIndex(QList<QVector<QString>> *messageList);
bool creatAllIndex(QList<QString> *messageList);
bool creatAllIndex(QQueue<QVector<QString>> *messageList);
bool creatAllIndex(QQueue<QString> *messageList);
bool deleteAllIndex(QStringList *pathlist);
private:
explicit IndexGenerator(bool rebuild = false,QObject *parent = nullptr);
static QMutex m_mutex;
//For file name index
void HandlePathList(QList<QVector<QString>> *messageList);
void HandlePathList(QQueue<QVector<QString> > *messageList);
//For file content index
void HandlePathList(QList<QString> *messageList);
void HandlePathList(QQueue<QString> *messageList);
static Document GenerateDocument(const QVector<QString> &list);
static Document GenerateContentDocument(const QString &list);
//add one data in database

24
libsearch/index/inotify-index.cpp
View File

@ -80,8 +80,8 @@ void InotifyIndex::run(){
ssize_t numRead;
QList<QVector<QString>>* indexList = new QList<QVector<QString>>();
QList<QString>* contentIndexList = new QList<QString>();
QQueue<QVector<QString>>* indexQueue = new QQueue<QVector<QString>>();
QQueue<QString>* contentIndexQueue = new QQueue<QString>();
for (;;) { /* Read events forever */
numRead = read(m_fd, buf, BUF_LEN);
@ -110,14 +110,14 @@ void InotifyIndex::run(){
/*--------------------------------*/
// IndexGenerator::getInstance()->creatAllIndex(QQueue<QVector<QString>>(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0")));
indexList->append(QVector<QString>() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0"));
IndexGenerator::getInstance()->creatAllIndex(indexList);
indexList->clear();
indexQueue->enqueue(QVector<QString>() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0"));
IndexGenerator::getInstance()->creatAllIndex(indexQueue);
indexQueue->clear();
for (auto i : this->targetFileTypeVec){
if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){
contentIndexList->append(QString(currentPath[event->wd] + '/' + event->name));
IndexGenerator::getInstance()->creatAllIndex(contentIndexList);
contentIndexList->clear();
contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name));
IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue);
contentIndexQueue->clear();
}
}
/*--------------------------------*/
@ -134,8 +134,8 @@ void InotifyIndex::run(){
}
}
delete indexList;
indexList = nullptr;
delete contentIndexList;
contentIndexList = nullptr;
delete indexQueue;
indexQueue = nullptr;
delete contentIndexQueue;
contentIndexQueue = nullptr;
}

57
src/main.cpp
View File

@ -92,12 +92,30 @@ int main(int argc, char *argv[])
{
qInstallMessageHandler(messageOutput);
QApplication::setAttribute(Qt::AA_EnableHighDpiScaling);
QApplication::setAttribute(Qt::AA_UseHighDpiPixmaps);
qDebug() << "main start";
FirstIndex fi("/home");
fi.start();
InotifyIndex ii("/home");
ii.start();
QtSingleApplication app("ukui-search", argc, argv);
app.setQuitOnLastWindowClosed(false);
if(app.isRunning())
{
app.sendMessage(QApplication::arguments().length() > 1 ? QApplication::arguments().at(1) : app.applicationFilePath());
qDebug() << QObject::tr("ukui-search is already running!");
return EXIT_SUCCESS;
}/*else {
QCommandLineParser parser;
QCommandLineOption debugOption({"d", "debug"}, QObject::tr("Display debug information"));
QCommandLineOption showsearch({"s", "show"}, QObject::tr("show search widget"));
parser.addOptions({debugOption, showsearch});
parser.process(app);
}*/
// qDebug() << "main start";
// FirstIndex fi("/home");
// fi.start();
// InotifyIndex ii("/home");
// ii.start();
/*-------------ukuisearchdbus Test start-----------------*/
// UkuiSearchQDBus usQDBus;
// usQDBus.setInotifyMaxUserWatches();
@ -131,27 +149,6 @@ int main(int argc, char *argv[])
// search->onKeywordSearchContent("g,e,x");
/*-------------文本搜索 Test End-----------------*/
QApplication::setAttribute(Qt::AA_EnableHighDpiScaling);
QApplication::setAttribute(Qt::AA_UseHighDpiPixmaps);
QtSingleApplication app("ukui-search", argc, argv);
app.setQuitOnLastWindowClosed(false);
qDebug() << "main start x2";
if(app.isRunning())
{
app.sendMessage(QApplication::arguments().length() > 1 ? QApplication::arguments().at(1) : app.applicationFilePath());
qDebug() << QObject::tr("ukui-search is already running!");
return EXIT_SUCCESS;
}/*else {
QCommandLineParser parser;
QCommandLineOption debugOption({"d", "debug"}, QObject::tr("Display debug information"));
QCommandLineOption showsearch({"s", "show"}, QObject::tr("show search widget"));
parser.addOptions({debugOption, showsearch});
parser.process(app);
}*/
// 加载国际化文件
QTranslator translator;
try {
@ -164,15 +161,15 @@ int main(int argc, char *argv[])
MainWindow *w = new MainWindow;
QStringList arguments = QCoreApplication::arguments();
centerToScreen(w);
w->show();
w->raise();
w->activateWindow();
// w->loadMainWindow();
app.setActivationWindow(w);
// if(arguments.size()>1)
// w->searchContent(arguments.at(1));
QObject::connect(&app, SIGNAL(messageReceived(const QString&)),w, SLOT(bootOptionsFilter(const QString&)));
// qDebug() << "main start";
// FirstIndex* fi = new FirstIndex("/home");
// fi->start();
return app.exec();
}

14
src/mainwindow.cpp
View File

@ -194,6 +194,8 @@ void MainWindow::bootOptionsFilter(QString opt)
if (opt == "-s" || opt == "-show") {
clearSearchResult();
this->show();
this->raise();
this->activateWindow();
}
}
@ -251,7 +253,17 @@ void MainWindow::searchContent(QString searchcontent){
//文件、文件夹、内容搜索
FileSearcher *search = new FileSearcher();
connect(search, &FileSearcher::resultDir, this, [ = ](QQueue<QString> * dirQueue) {
qWarning()<<"dirFile---";
qWarning()<<"resultDir---";
QString firstDir;
while(1)
{
if(!dirQueue->isEmpty()){
firstDir = dirQueue->at(0);
qWarning()<<"firstDir"<<firstDir;
break;
}
}
});
connect(search, &FileSearcher::resultFile, this, [ = ](QQueue<QString> * fileQueue) {
qWarning()<<"resultFile---";