openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Optimize index switch logic.

This commit is contained in:
iaom 2021-11-09 10:07:41 +08:00
parent 311983d9b2
commit e95136e6bb
10 changed files with 13 additions and 518 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libsearch/file-utils.cpp
View File

@ -29,7 +29,7 @@
using namespace Zeeker;
size_t FileUtils::_max_index_count = 0;
size_t FileUtils::_current_index_count = 0;
unsigned short FileUtils::_index_status = 0;
unsigned short FileUtils::indexStatus = 0;
FileUtils::SearchMethod FileUtils::searchMethod = FileUtils::SearchMethod::DIRECTSEARCH;
QMap<QString, QStringList> FileUtils::map_chinese2pinyin = QMap<QString, QStringList>();
static QMutex iconMutex;

2
libsearch/file-utils.h
View File

@ -100,7 +100,7 @@ public:
static bool isEncrypedOrUnreadable(QString path);
static size_t _max_index_count;
static size_t _current_index_count; //this one has been Abandoned,do not use it.
static unsigned short _index_status;
static unsigned short indexStatus;
enum class SearchMethod { DIRECTSEARCH = 0, INDEXSEARCH = 1};
static SearchMethod searchMethod;

4
libsearch/index/first-index.cpp
View File

@ -131,7 +131,7 @@ void FirstIndex::run() {
assert(false);
}
++FileUtils::_index_status;
++FileUtils::indexStatus;
pid_t pid;
pid = fork();
if(pid == 0) {
@ -236,7 +236,7 @@ void FirstIndex::run() {
qWarning() << "First Index fork error!!";
} else {
waitpid(pid, NULL, 0);
--FileUtils::_index_status;
--FileUtils::indexStatus;
}
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2");

2
libsearch/index/index.pri
View File

@ -8,7 +8,6 @@ HEADERS += \
$$PWD/first-index.h \
$$PWD/index-generator.h \
$$PWD/index-status-recorder.h \
$$PWD/inotify-index.h \
$$PWD/inotify-watch.h \
$$PWD/pending-file-queue.h \
$$PWD/pending-file.h \
@ -25,7 +24,6 @@ SOURCES += \
$$PWD/first-index.cpp \
$$PWD/index-generator.cpp \
$$PWD/index-status-recorder.cpp \
$$PWD/inotify-index.cpp \
$$PWD/inotify-watch.cpp \
$$PWD/pending-file-queue.cpp \
$$PWD/pending-file.cpp \

418
libsearch/index/inotify-index.cpp
View File

@ -1,418 +0,0 @@
/*
* Copyright (C) 2020, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangzihao <zhangzihao@kylinos.cn>
*
*/
#include "inotify-index.h"
#include <QDataStream>
#define CREATE_FILE_NAME_INDEX \
indexQueue->enqueue(QVector<QString>() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); \
IndexGenerator::getInstance()->creatAllIndex(indexQueue); \
indexQueue->clear();
#define CREATE_FILE_CONTENT_INDEX \
if ((!QString(event->name).split(".").isEmpty()) && (true == this->targetFileTypeMap[QString(event->name).split(".").last()])) { \
contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); \
IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); \
contentIndexQueue->clear(); \
break; \
}
#define TRAVERSE_DIR \
QString tmp = currentPath[event->wd] + '/' + event->name; \
QFileInfo fi(tmp); \
if(!fi.isSymLink()){ \
AddWatch(tmp); \
setPath(tmp); \
Traverse(); \
}
#define CREATE_FILE \
CREATE_FILE_NAME_INDEX \
CREATE_FILE_CONTENT_INDEX
using namespace Zeeker;
InotifyIndex::InotifyIndex(const QString& path) : Traverse_BFS(path) {
qDebug() << "setInotifyMaxUserWatches start";
UkuiSearchQDBus usQDBus;
usQDBus.setInotifyMaxUserWatches();
qDebug() << "setInotifyMaxUserWatches end";
m_sharedMemory = new QSharedMemory("ukui-search-shared-map", this);
}
InotifyIndex::~InotifyIndex() {
IndexGenerator::getInstance()->~IndexGenerator();
qWarning() << "~InotifyIndex";
}
void InotifyIndex::firstTraverse() {
QQueue<QString> bfs;
bfs.enqueue(this->path);
QFileInfoList list;
QDir dir;
dir.setFilter(QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot);
dir.setSorting(QDir::DirsFirst);
while(!bfs.empty()) {
dir.setPath(bfs.dequeue());
list = dir.entryInfoList();
for(auto i : list) {
if(i.isDir() && (!(i.isSymLink()))) {
this->AddWatch(i.absoluteFilePath());
bfs.enqueue(i.absoluteFilePath());
}
}
}
}
void InotifyIndex::DoSomething(const QFileInfo& fileInfo) {
qDebug() << fileInfo.fileName() << "-------" << fileInfo.absoluteFilePath();
if(fileInfo.isDir() && (!fileInfo.isSymLink())) {
this->AddWatch(fileInfo.absoluteFilePath());
}
QQueue<QVector<QString> > tempFile;
tempFile.enqueue(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString((fileInfo.isDir() && (!fileInfo.isSymLink())) ? "1" : "0"));
IndexGenerator::getInstance()->creatAllIndex(&tempFile);
if((fileInfo.fileName().split(".", QString::SkipEmptyParts).length() > 1) && (true == targetFileTypeMap[fileInfo.fileName().split(".").last()])) {
QQueue<QString> tmp;
tmp.enqueue(fileInfo.absoluteFilePath());
IndexGenerator::getInstance()->creatAllIndex(&tmp);
}
}
bool InotifyIndex::AddWatch(const QString &path) {
int ret = inotify_add_watch(m_fd, path.toStdString().c_str(), (IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | IN_DELETE | IN_MODIFY));
if(ret == -1) {
qDebug() << "AddWatch error:" << path;
return false;
}
// Q_ASSERT(ret != -1);
assert(ret != -1);
currentPath[ret] = path;
// qDebug() << "Watch: " << path << "ret: " << ret;
return true;
}
bool InotifyIndex::RemoveWatch(const QString &path, bool removeFromDatabase) {
int ret = inotify_rm_watch(m_fd, currentPath.key(path));
if(ret) {
qDebug() << "remove path error";
return false;
}
// Q_ASSERT(ret == 0);
assert(ret == 0);
if(removeFromDatabase) {
for(QMap<int, QString>::Iterator i = currentPath.begin(); i != currentPath.end();) {
// qDebug() << i.value();
if(i.value().length() > path.length()) {
// if (i.value().mid(0, path.length()) == path){
// if (path.startsWith(i.value())){
if(i.value().startsWith(path)) {
qDebug() << "remove path: " << i.value();
ret = inotify_rm_watch(m_fd, currentPath.key(path));
if(ret) {
qDebug() << "remove path error";
// return false;
}
// assert(ret == 0);
/*--------------------------------*/
//在此调用删除索引
qDebug() << i.value();
IndexGenerator::getInstance()->deleteAllIndex(new QStringList(i.value()));
/*--------------------------------*/
currentPath.erase(i++);
// i++;
} else {
i++;
}
} else {
i++;
}
}
} else {
for(QMap<int, QString>::Iterator i = currentPath.begin(); i != currentPath.end();) {
// qDebug() << i.value();
if(i.value().length() > path.length()) {
// if (i.value().mid(0, path.length()) == path){
// if (path.startsWith(i.value())){
if(i.value().startsWith(path)) {
qDebug() << "remove path: " << i.value();
ret = inotify_rm_watch(m_fd, currentPath.key(path));
if(ret) {
qDebug() << "remove path error";
// return false;
}
// assert(ret == 0);
currentPath.erase(i++);
// i++;
} else {
i++;
}
} else {
i++;
}
}
}
// qDebug() << path;
//这个貌似不用删，先mark一下
// currentPath.remove(currentPath.key(path));
return true;
}
void InotifyIndex::eventProcess(const char* buf, ssize_t tmp) {
QQueue<QVector<QString>>* indexQueue = new QQueue<QVector<QString>>();
QQueue<QString>* contentIndexQueue = new QQueue<QString>();
ssize_t numRead = 0;
numRead = tmp;
char * p = const_cast<char*>(buf);
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "0");
for(; p < buf + numRead;) {
struct inotify_event * event = reinterpret_cast<inotify_event *>(p);
if(event->name[0] != '.') {
qDebug() << "Read Event event->wd: " << event->wd;
qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
qDebug() << QString(currentPath[event->wd] + '/' + event->name);
// switch (event->mask) {
if(event->mask & IN_CREATE) {
//Create top dir first, traverse it last.
qDebug() << "IN_CREATE";
CREATE_FILE
if(event->mask & IN_ISDIR) {
TRAVERSE_DIR
}
goto next;
}
if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)) {
qDebug() << "IN_DELETE or IN_MOVED_FROM";
if(event->mask & IN_ISDIR) {
RemoveWatch(currentPath[event->wd] + '/' + event->name);
}
//delete once more
IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
goto next;
}
if(event->mask & IN_MODIFY) {
qDebug() << "IN_MODIFY";
if(!(event->mask & IN_ISDIR)) {
// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
CREATE_FILE
}
goto next;
}
if(event->mask & IN_MOVED_TO) {
qDebug() << "IN_MOVED_TO";
if(event->mask & IN_ISDIR) {
RemoveWatch(currentPath[event->wd] + '/' + event->name);
// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
CREATE_FILE
TRAVERSE_DIR
} else {
IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
CREATE_FILE
}
goto next;
}
}
next:
p += sizeof(struct inotify_event) + event->len;
}
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2");
delete indexQueue;
indexQueue = nullptr;
delete contentIndexQueue;
contentIndexQueue = nullptr;
}
void InotifyIndex::run() {
m_fd = inotify_init();
qDebug() << "m_fd----------->" << m_fd;
this->AddWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation));
this->setPath(QStandardPaths::writableLocation(QStandardPaths::HomeLocation));
this->firstTraverse();
int fifo_fd;
char buffer[2];
memset(buffer, 0, sizeof(buffer));
fifo_fd = open(UKUI_SEARCH_PIPE_PATH, O_RDWR);
if(fifo_fd == -1) {
perror("open fifo error\n");
assert(false);
}
int retval = read(fifo_fd, buffer, sizeof(buffer));
if(retval == -1) {
perror("read error\n");
assert(false);
}
printf("read fifo=[%s]\n", buffer);
printf("read data ok\n");
close(fifo_fd);
if(buffer[0] & 0x1) {
printf("data confirmed\n");
}
unlink(UKUI_SEARCH_PIPE_PATH);
char buf[BUF_LEN] __attribute__((aligned(8)));
ssize_t numRead;
while(FileUtils::SearchMethod::INDEXSEARCH == FileUtils::searchMethod) {
// for (;;) { /* Read events forever */
memset(buf, 0x00, BUF_LEN);
numRead = read(m_fd, buf, BUF_LEN);
if(numRead == -1) {
printf("\033[1;31;40mread event error\033[0m\n");
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
fflush(stdout);
assert(false);
}
//TODO: Merge multiple signals.
char * tmp = const_cast<char*>(buf);
for(; tmp < buf + numRead;) {
struct inotify_event * event = reinterpret_cast<inotify_event *>(tmp);
if(event->name[0] != '.') {
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
// qDebug("mask:0x%x,",event->mask);
break;
}
tmp += sizeof(struct inotify_event) + event->len;
}
if(tmp >= buf + numRead) {
continue;
}
++FileUtils::_index_status;
pid_t pid;
pid = fork();
if(pid == 0) {
prctl(PR_SET_PDEATHSIG, SIGTERM);
prctl(PR_SET_NAME, "inotify-index");
if(numRead == 0) {
qDebug() << "read() from inotify fd returned 0!";
}
if(numRead == -1) {
qDebug() << "read";
}
eventProcess(buf, numRead);
fd_set read_fds;
int rc;
timeval* read_timeout = (timeval*)malloc(sizeof(timeval));
read_timeout->tv_sec = 40;
read_timeout->tv_usec = 0;
for(;;) {
FD_ZERO(&read_fds);
FD_SET(m_fd, &read_fds);
qDebug() << read_timeout->tv_sec;
rc = select(m_fd + 1, &read_fds, NULL, NULL, read_timeout);
if(rc < 0) {
// error
qWarning() << "select result < 0, error!";
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
assert(false);
} else if(rc == 0) {
qDebug() << "select timeout!";
::free(read_timeout);
IndexGenerator::getInstance()->~IndexGenerator();
QBuffer buffer;
QDataStream out(&buffer);
if (m_sharedMemory->isAttached()) {
m_sharedMemory->detach();
}
buffer.open(QBuffer::ReadWrite);
out << currentPath;
int size = buffer.size();
if (!m_sharedMemory->create(size)) {
qDebug() << "Create sharedMemory Error: " << m_sharedMemory->errorString();
} else {
m_sharedMemory->lock();
char *to = static_cast<char *>(m_sharedMemory->data());
const char *from = buffer.data().constData();
memcpy(to, from, qMin(size, m_sharedMemory->size()));
m_sharedMemory->unlock();
}
// GlobalSettings::getInstance()->forceSync();
::_exit(0);
} else {
memset(buf, 0x00, BUF_LEN);
numRead = read(m_fd, buf, BUF_LEN);
if(numRead == -1) {
printf("\033[1;31;40mread event error\033[0m\n");
fflush(stdout);
assert(false);
}
//TODO: Merge multiple signals.
char * tmp = const_cast<char*>(buf);
for(; tmp < buf + numRead; ) {
struct inotify_event * event = reinterpret_cast<inotify_event *>(tmp);
if(event->name[0] != '.') {
break;
}
tmp += sizeof(struct inotify_event) + event->len;
}
if(tmp >= buf + numRead) {
continue;
}
qDebug() << "Read " << numRead << " bytes from inotify fd";
this->eventProcess(buf, numRead);
}
}
} else if(pid > 0) {
memset(buf, 0x00, BUF_LEN);
waitpid(pid, NULL, 0);
if (!m_sharedMemory->attach()) {
qDebug() << "SharedMemory attach Error: " << m_sharedMemory->errorString();
} else {
QBuffer buffer;
QDataStream in(&buffer);
QMap<int, QString> pathMap;
m_sharedMemory->lock();
buffer.setData(static_cast<const char *>(m_sharedMemory->constData()), m_sharedMemory->size());
buffer.open(QBuffer::ReadWrite);
in >> pathMap;
m_sharedMemory->unlock();
m_sharedMemory->detach();
currentPath = pathMap;
}
--FileUtils::_index_status;
} else {
assert(false);
}
}
if(FileUtils::SearchMethod::DIRECTSEARCH == FileUtils::searchMethod) {
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "3");
RemoveWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation), false);
}
}

86
libsearch/index/inotify-index.h
View File

@ -1,86 +0,0 @@
/*
* Copyright (C) 2020, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangzihao <zhangzihao@kylinos.cn>
*
*/
#ifndef INOTIFYINDEX_H
#define INOTIFYINDEX_H
#include <QThread>
#include <QTimer>
#include <QSharedMemory>
#include <unistd.h>
#include <sys/inotify.h>
#include "index-generator.h"
#include "traverse_bfs.h"
#include "ukui-search-qdbus.h"
#include "index-status-recorder.h"
#include "file-utils.h"
#include "first-index.h"
#include "common.h"
#define BUF_LEN 1024000
namespace Zeeker {
class InotifyIndex;
static InotifyIndex* global_instance_of_index = nullptr;
class InotifyIndex : public QThread, public Traverse_BFS {
Q_OBJECT
public:
static InotifyIndex* getInstance(const QString& path) {
if(!global_instance_of_index) {
global_instance_of_index = new InotifyIndex(path);
}
return global_instance_of_index;
}
InotifyIndex(const QString&);
~InotifyIndex();
bool AddWatch(const QString&);
bool RemoveWatch(const QString&, bool removeFromDatabase = true);
virtual void DoSomething(const QFileInfo &) final;
void eventProcess(const char*, ssize_t);
void firstTraverse();
protected:
void run() override;
private:
int m_fd;
QMap<int, QString> currentPath;
const QMap<QString, bool> targetFileTypeMap = {
std::map<QString, bool>::value_type("doc", true),
std::map<QString, bool>::value_type("docx", true),
std::map<QString, bool>::value_type("ppt", true),
std::map<QString, bool>::value_type("pptx", true),
std::map<QString, bool>::value_type("xls", true),
std::map<QString, bool>::value_type("xlsx", true),
std::map<QString, bool>::value_type("txt", true),
std::map<QString, bool>::value_type("dot", true),
std::map<QString, bool>::value_type("wps", true),
std::map<QString, bool>::value_type("pps", true),
std::map<QString, bool>::value_type("dps", true),
std::map<QString, bool>::value_type("et", true),
std::map<QString, bool>::value_type("pdf", true)
};
QSharedMemory *m_sharedMemory = nullptr;
};
}
#endif // INOTIFYINDEX_H

5
libsearch/index/inotify-watch.cpp
View File

@ -180,10 +180,11 @@ void InotifyWatch::run()
int rc;
rc = select(m_inotifyFd + 1, &fds, NULL, NULL, NULL);
if(rc > 0) {
++FileUtils::_index_status;
++FileUtils::indexStatus;
int avail;
if (ioctl(m_inotifyFd, FIONREAD, &avail) == EINVAL) {
qWarning() << "Did not receive an entire inotify event.";
--FileUtils::indexStatus;
return;
}
@ -211,7 +212,7 @@ void InotifyWatch::run()
slotEvent(buf, len);
free(buf);
}
--FileUtils::_index_status;
--FileUtils::indexStatus;
} else if(rc < 0) {
// error
qWarning() << "select result < 0, error!";

2
libsearch/index/searchmethodmanager.cpp
View File

@ -12,7 +12,7 @@ void SearchMethodManager::searchMethod(FileUtils::SearchMethod sm) {
} else {
qWarning("enum class error!!!\n");
}
if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::_index_status) {
if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::indexStatus) {
// Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running.
QDir fifoDir = QDir(QDir::homePath() + "/.config/org.ukui/ukui-search");

2
src/main.cpp
View File

@ -131,7 +131,7 @@ void searchMethod(FileUtils::SearchMethod sm){
printf("enum class error!!!\n");
qWarning("enum class error!!!\n");
}
if (FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::_index_status) {
if (FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::indexStatus) {
qWarning() << "start first index";
FirstIndex fi("/home/zhangzihao/Desktop");
fi.start();

8
src/settings-widget.cpp
View File

@ -272,8 +272,8 @@ void SettingsWidget::clearLayout(QLayout * layout) {
* @brief SettingsWidget::refreshIndexState 定时刷新索引项
*/
void SettingsWidget::refreshIndexState() {
// qDebug()<<"FileUtils::_index_status: "<<FileUtils::_index_status;
if(FileUtils::_index_status != 0) {
// qDebug()<<"FileUtils::indexStatus: "<<FileUtils::indexStatus;
if(FileUtils::indexStatus != 0) {
this->setIndexState(true);
} else {
this->setIndexState(false);
@ -281,8 +281,8 @@ void SettingsWidget::refreshIndexState() {
m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::_max_index_count)));
m_timer = new QTimer;
connect(m_timer, &QTimer::timeout, this, [ = ]() {
qDebug() << "FileUtils::_index_status: " << FileUtils::_index_status;
if(FileUtils::_index_status != 0) {
qDebug() << "FileUtils::indexStatus: " << FileUtils::indexStatus;
if(FileUtils::indexStatus != 0) {
this->setIndexState(true);
} else {
this->setIndexState(false);