openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Creat two sub-processes for file index to reduce the resource consumption(ongoing).

This commit is contained in:
zhangpengfei 2021-01-21 13:50:21 +08:00
parent 3efac5701c
commit 452896a354
10 changed files with 289 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
debian/changelog vendored
View File

@ -1,3 +1,9 @@
ukui-search (0.0.1+0120) v101; urgency=medium
* Bugs Fixed.
-- zhangpengfei <zhangpengfei@kylinos.cn> Wed, 20 Jan 2021 15:55:57 +0800
ukui-search (0.0.1+0118) v101; urgency=medium
* Feature: Add animation into inputbox.
@ -11,8 +17,7 @@ ukui-search (0.0.1+0118) v101; urgency=medium
* Fix: Translation is incompleted.(33047)
* Fix: Widget crashed when searching & open file/filepath failed.
-- zhangjiaping <zhangjiaping@zhangpengfei> Mon, 18 Jan 2021 14:3
6:12 +0800
-- zhangjiaping <zhangjiaping@kylinos.cn> Mon, 18 Jan 2021 14:31:24 +0800
ukui-search (0.0.1+0115) v101; urgency=medium
@ -24,7 +29,7 @@ ukui-search (0.0.1+0114) v101; urgency=medium
* Bugs fixed.
-- zhangpengfei <zhangpengfei@zhangpengfei> Thu, 14 Jan 2021 20:47:42 +0800
-- zhangpengfei <zhangpengfei@kylinos.cn> Thu, 14 Jan 2021 20:47:42 +0800
ukui-search (0.0.1+0113) v101; urgency=medium

2
libchinese-segmentation/chinese-segmentation.cpp
View File

@ -6,7 +6,6 @@ QMutex ChineseSegmentation::m_mutex;
ChineseSegmentation::ChineseSegmentation()
{
QMutexLocker locker(&m_mutex);
const char * const DICT_PATH = "/usr/share/ukui-search/res/dict/jieba.dict.utf8";
const char * const HMM_PATH = "/usr/share/ukui-search/res/dict/hmm_model.utf8";
const char * const USER_DICT_PATH ="/usr/share/ukui-search/res/dict/user.dict.utf8";
@ -29,6 +28,7 @@ ChineseSegmentation::~ChineseSegmentation()
ChineseSegmentation *ChineseSegmentation::getInstance()
{
QMutexLocker locker(&m_mutex);
if (!global_instance_chinese_segmentation) {
global_instance_chinese_segmentation = new ChineseSegmentation;
}

1
libsearch/file-utils.h
View File

@ -10,6 +10,7 @@
#define CREATING_INDEX 1
#define FINISH_CREATING_INDEX 2
class LIBSEARCH_EXPORT FileUtils
{
public:

3
libsearch/index/file-searcher.cpp
View File

@ -66,6 +66,7 @@ void FileSearcher::onKeywordSearch(QString keyword,QQueue<QString> *searchResult
total += resultCount;
begin += num;
}
return;
});
// Q_EMIT this->resultFile(m_search_result_file);
//dir
@ -84,6 +85,7 @@ void FileSearcher::onKeywordSearch(QString keyword,QQueue<QString> *searchResult
total += resultCount;
begin += num;
}
return;
});
// Q_EMIT this->resultDir(m_search_result_dir);
//content
@ -103,6 +105,7 @@ void FileSearcher::onKeywordSearch(QString keyword,QQueue<QString> *searchResult
total += resultCount;
begin += num;
}
return;
});
// Q_EMIT this->resultContent(m_search_result_content);
}

25
libsearch/index/first-index.cpp
View File

@ -2,7 +2,6 @@
#include "first-index.h"
#include <QDebug>
void handler(int){
qDebug() << "Recieved SIGTERM!";
GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2");
@ -107,6 +106,11 @@ void FirstIndex::run(){
// this->p_indexGenerator->creatAllIndex(this->q_content_index);
pid_t pid;
pid = fork();
if(pid == 0)
{
prctl(PR_SET_NAME,"first-index");
FileUtils::_index_status = CREATING_INDEX;
QSemaphore sem(5);
QMutex mutex1, mutex2, mutex3;
@ -144,14 +148,16 @@ void FirstIndex::run(){
mutex1.unlock();
mutex2.unlock();
mutex3.unlock();
_exit(0);
// qDebug() << "first index end;";
// qDebug() << "first index end;";
//don't use it now!!!!
//MouseZhangZh
// this->~FirstIndex();
// qDebug() << "~FirstIndex end;";
// this->~FirstIndex();
// qDebug() << "~FirstIndex end;"
if (this->q_index)
@ -166,8 +172,15 @@ void FirstIndex::run(){
QThreadPool::globalInstance()->releaseThread();
QThreadPool::globalInstance()->waitForDone();
}
else if(pid < 0)
{
qWarning()<<"First Index fork error!!";
}
else
{
waitpid(pid,NULL,0);
}
FileUtils::_index_status = FINISH_CREATING_INDEX;
qDebug() << "sigset start!";

5
libsearch/index/first-index.h
View File

@ -4,6 +4,11 @@
#include <QThread>
#include <QtConcurrent/QtConcurrent>
#include <signal.h>
#include <QSemaphore>
#include<sys/types.h>
#include<unistd.h>
#include <sys/wait.h>
#include <sys/prctl.h>
//#include <QtConcurrent>
#include "traverse_bfs.h"
#include "global-settings.h"

4
libsearch/index/index-generator.cpp
View File

@ -450,7 +450,11 @@ bool IndexGenerator::deleteAllIndex(QStringList *pathlist)
qDebug()<<"delete path"<<doc;
qDebug()<<"delete md5"<<QString::fromStdString(uniqueterm);
m_database_path->commit();
m_database_content->commit();
qDebug()<< "--delete finish--";
// qDebug()<<"m_database_path->get_lastdocid()!!!"<<m_database_path->get_lastdocid();
// qDebug()<<"m_database_path->get_doccount()!!!"<<m_database_path->get_doccount();
}
catch(const Xapian::Error &e)
{

187
libsearch/index/inotify-index.cpp
View File

@ -1,6 +1,5 @@
#include "inotify-index.h"
InotifyIndex::InotifyIndex(const QString& path) : Traverse_BFS(path)
{
/*-------------ukuisearchdbus Test start-----------------*/
@ -88,42 +87,27 @@ bool InotifyIndex::RemoveWatch(const QString &path){
return true;
}
/*
* Symbolic Link!!!!!!!!!!!!!!!!!!
* Sysmbolic link to database dir will make a Infinite loop !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
* MouseZhangZh
*/
void InotifyIndex::eventProcess(const char* buf, ssize_t tmp){
// qDebug() << "Read " << numRead << " bytes from inotify fd";
void InotifyIndex::run(){
/* Process all of the events in buffer returned by read() */
char * p;
char buf[BUF_LEN] __attribute__((aligned(8)));
ssize_t numRead;
QQueue<QVector<QString>>* indexQueue = new QQueue<QVector<QString>>();
QQueue<QString>* contentIndexQueue = new QQueue<QString>();
for (;;) { /* Read events forever */
numRead = read(m_fd, buf, BUF_LEN);
if (numRead == 0) {
qDebug() << "read() from inotify fd returned 0!";
}
if (numRead == -1) {
qDebug() << "read";
}
// qDebug() << "Read " << numRead << " bytes from inotify fd";
ssize_t numRead = 0;
numRead = tmp;
char * p = const_cast<char*>(buf);
/* Process all of the events in buffer returned by read() */
for (p = buf; p < buf + numRead;) {
for (; p < buf + numRead;) {
struct inotify_event * event = reinterpret_cast<inotify_event *>(p);
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
if(event->name[0] != '.'){
qDebug() << QString(currentPath[event->wd] + '/' + event->name);
FileUtils::_index_status = CREATING_INDEX;
// switch (event->mask) {
// switch (event->mask) {
if (event->mask & IN_CREATE){
if (event->mask & IN_ISDIR){
AddWatch(currentPath[event->wd] + '/' + event->name);
@ -132,7 +116,7 @@ void InotifyIndex::run(){
}
/*--------------------------------*/
// IndexGenerator::getInstance()->creatAllIndex(QQueue<QVector<QString>>(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0")));
// IndexGenerator::getInstance()->creatAllIndex(QQueue<QVector<QString>>(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0")));
indexQueue->enqueue(QVector<QString>() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0"));
IndexGenerator::getInstance()->creatAllIndex(indexQueue);
indexQueue->clear();
@ -185,10 +169,6 @@ void InotifyIndex::run(){
this->setPath(currentPath[event->wd] + '/' + event->name);
Traverse();
//
indexQueue->enqueue(QVector<QString>() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0"));
IndexGenerator::getInstance()->creatAllIndex(indexQueue);
indexQueue->clear();
@ -218,64 +198,119 @@ void InotifyIndex::run(){
goto next;
}
// }
// }
// //传创建或移动过来的文件路径
// if((event->mask & IN_CREATE)){
// //添加监视要先序遍历，先添加top节点
// if (event->mask & IN_ISDIR){
// AddWatch(currentPath[event->wd] + '/' + event->name);
// this->setPath(currentPath[event->wd] + '/' + event->name);
// Traverse();
// }
// //传创建或移动过来的文件路径
// if((event->mask & IN_CREATE)){
// //添加监视要先序遍历，先添加top节点
// if (event->mask & IN_ISDIR){
// AddWatch(currentPath[event->wd] + '/' + event->name);
// this->setPath(currentPath[event->wd] + '/' + event->name);
// Traverse();
// }
// /*--------------------------------*/
//// IndexGenerator::getInstance()->creatAllIndex(QQueue<QVector<QString>>(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0")));
// indexQueue->enqueue(QVector<QString>() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0"));
// IndexGenerator::getInstance()->creatAllIndex(indexQueue);
// indexQueue->clear();
// for (auto i : this->targetFileTypeVec){
// if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){
// contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name));
// IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue);
// contentIndexQueue->clear();
// break;
// }
// }
// /*--------------------------------*/
// }
// else if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){
// if (event->mask & IN_ISDIR){
// RemoveWatch(currentPath[event->wd] + '/' + event->name);
// }
// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
// }
// else if((event->mask & IN_MODIFY) | (event->mask & IN_MOVED_TO)){
// if (!(event->mask & IN_ISDIR)){
// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
// indexQueue->enqueue(QVector<QString>() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0"));
// IndexGenerator::getInstance()->creatAllIndex(indexQueue);
// indexQueue->clear();
// for (auto i : this->targetFileTypeVec){
// if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){
// contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name));
// IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue);
// contentIndexQueue->clear();
// break;
// }
// }
// }
// }
// /*--------------------------------*/
//// IndexGenerator::getInstance()->creatAllIndex(QQueue<QVector<QString>>(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0")));
// indexQueue->enqueue(QVector<QString>() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0"));
// IndexGenerator::getInstance()->creatAllIndex(indexQueue);
// indexQueue->clear();
// for (auto i : this->targetFileTypeVec){
// if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){
// contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name));
// IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue);
// contentIndexQueue->clear();
// break;
// }
// }
// /*--------------------------------*/
// }
// else if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){
// if (event->mask & IN_ISDIR){
// RemoveWatch(currentPath[event->wd] + '/' + event->name);
// }
// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
// }
// else if((event->mask & IN_MODIFY) | (event->mask & IN_MOVED_TO)){
// if (!(event->mask & IN_ISDIR)){
// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
// indexQueue->enqueue(QVector<QString>() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0"));
// IndexGenerator::getInstance()->creatAllIndex(indexQueue);
// indexQueue->clear();
// for (auto i : this->targetFileTypeVec){
// if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){
// contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name));
// IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue);
// contentIndexQueue->clear();
// break;
// }
// }
// }
// }
/*--------------------------------*/
FileUtils::_index_status = FINISH_CREATING_INDEX;
}
next:
p += sizeof(struct inotify_event) + event->len;
}
}
delete indexQueue;
indexQueue = nullptr;
delete contentIndexQueue;
contentIndexQueue = nullptr;
}
/*
* Symbolic Link!!!!!!!!!!!!!!!!!!
* Sysmbolic link to database dir will make a Infinite loop !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
* MouseZhangZh
*/
void InotifyIndex::run(){
char buf[BUF_LEN] __attribute__((aligned(8)));
ssize_t numRead;
for (;;) { /* Read events forever */
numRead = read(m_fd, buf, BUF_LEN);
pid_t pid;
pid = fork();
if(pid == 0)
{
prctl(PR_SET_NAME,"inotify-index");
if (numRead == 0) {
qDebug() << "read() from inotify fd returned 0!";
}
if (numRead == -1) {
qDebug() << "read";
}
eventProcess(buf, numRead);
QTimer* liveTime = new QTimer(this);
bool b_timeout = false;
liveTime->setInterval(30000);
connect(liveTime, &QTimer::timeout, this, [&](){
// _exit(0);
b_timeout = true;
});
liveTime->start();
for (;;){
numRead = read(m_fd, buf, BUF_LEN);
liveTime->stop();
this->eventProcess(buf, numRead);
if (b_timeout){
_exit(0);
}
liveTime->start();
}
}
else if (pid > 0){
memset(buf, 0x00, BUF_LEN);
waitpid(pid, NULL, 0);
}
else{
assert(false);
}
}
}

4
libsearch/index/inotify-index.h
View File

@ -2,6 +2,7 @@
#define INOTIFYINDEX_H
#include <QThread>
#include <QTimer>
#include <unistd.h>
#include <sys/inotify.h>
#include "index-generator.h"
@ -9,6 +10,7 @@
#include "ukui-search-qdbus.h"
#include "global-settings.h"
#include "file-utils.h"
#include "first-index.h"
#define BUF_LEN 1024
class InotifyIndex;
@ -29,6 +31,8 @@ public:
bool AddWatch(const QString&);
bool RemoveWatch(const QString&);
virtual void DoSomething(const QFileInfo &) final;
void eventProcess(const char*, ssize_t);
protected:
void run() override;
private:

2
src/main.cpp
View File

@ -26,6 +26,7 @@
#include <X11/Xlib.h>
#include <syslog.h>
#include <QObject>
#include <QSemaphore>
#include "qt-single-application.h"
#include "qt-local-peer.h"
//#include "inotify-manager.h"
@ -185,6 +186,7 @@ int main(int argc, char *argv[])
// FirstIndex* fi = new FirstIndex("/home/zhangzihao/Desktop/qwerty");
FirstIndex fi("/home");
fi.start();
fi.wait();
// fi->wait();
// fi->exit();
// delete fi;