From e95136e6bb8b1440e7d6f5a763dff924138642e0 Mon Sep 17 00:00:00 2001 From: iaom Date: Tue, 9 Nov 2021 10:07:41 +0800 Subject: [PATCH] Optimize index switch logic. --- libsearch/file-utils.cpp | 2 +- libsearch/file-utils.h | 2 +- libsearch/index/first-index.cpp | 4 +- libsearch/index/index.pri | 2 - libsearch/index/inotify-index.cpp | 418 ------------------------ libsearch/index/inotify-index.h | 86 ----- libsearch/index/inotify-watch.cpp | 5 +- libsearch/index/searchmethodmanager.cpp | 2 +- src/main.cpp | 2 +- src/settings-widget.cpp | 8 +- 10 files changed, 13 insertions(+), 518 deletions(-) delete mode 100644 libsearch/index/inotify-index.cpp delete mode 100644 libsearch/index/inotify-index.h diff --git a/libsearch/file-utils.cpp b/libsearch/file-utils.cpp index 32e2e25..d0afb38 100644 --- a/libsearch/file-utils.cpp +++ b/libsearch/file-utils.cpp @@ -29,7 +29,7 @@ using namespace Zeeker; size_t FileUtils::_max_index_count = 0; size_t FileUtils::_current_index_count = 0; -unsigned short FileUtils::_index_status = 0; +unsigned short FileUtils::indexStatus = 0; FileUtils::SearchMethod FileUtils::searchMethod = FileUtils::SearchMethod::DIRECTSEARCH; QMap FileUtils::map_chinese2pinyin = QMap(); static QMutex iconMutex; diff --git a/libsearch/file-utils.h b/libsearch/file-utils.h index b7825df..833f6a3 100644 --- a/libsearch/file-utils.h +++ b/libsearch/file-utils.h @@ -100,7 +100,7 @@ public: static bool isEncrypedOrUnreadable(QString path); static size_t _max_index_count; static size_t _current_index_count; //this one has been Abandoned,do not use it. - static unsigned short _index_status; + static unsigned short indexStatus; enum class SearchMethod { DIRECTSEARCH = 0, INDEXSEARCH = 1}; static SearchMethod searchMethod; diff --git a/libsearch/index/first-index.cpp b/libsearch/index/first-index.cpp index 52c3258..fc649eb 100644 --- a/libsearch/index/first-index.cpp +++ b/libsearch/index/first-index.cpp @@ -131,7 +131,7 @@ void FirstIndex::run() { assert(false); } - ++FileUtils::_index_status; + ++FileUtils::indexStatus; pid_t pid; pid = fork(); if(pid == 0) { @@ -236,7 +236,7 @@ void FirstIndex::run() { qWarning() << "First Index fork error!!"; } else { waitpid(pid, NULL, 0); - --FileUtils::_index_status; + --FileUtils::indexStatus; } IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2"); diff --git a/libsearch/index/index.pri b/libsearch/index/index.pri index 0b74560..bd7c705 100644 --- a/libsearch/index/index.pri +++ b/libsearch/index/index.pri @@ -8,7 +8,6 @@ HEADERS += \ $$PWD/first-index.h \ $$PWD/index-generator.h \ $$PWD/index-status-recorder.h \ - $$PWD/inotify-index.h \ $$PWD/inotify-watch.h \ $$PWD/pending-file-queue.h \ $$PWD/pending-file.h \ @@ -25,7 +24,6 @@ SOURCES += \ $$PWD/first-index.cpp \ $$PWD/index-generator.cpp \ $$PWD/index-status-recorder.cpp \ - $$PWD/inotify-index.cpp \ $$PWD/inotify-watch.cpp \ $$PWD/pending-file-queue.cpp \ $$PWD/pending-file.cpp \ diff --git a/libsearch/index/inotify-index.cpp b/libsearch/index/inotify-index.cpp deleted file mode 100644 index 04351f1..0000000 --- a/libsearch/index/inotify-index.cpp +++ /dev/null @@ -1,418 +0,0 @@ -/* - * Copyright (C) 2020, KylinSoft Co., Ltd. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * Authors: zhangzihao - * - */ -#include "inotify-index.h" -#include - -#define CREATE_FILE_NAME_INDEX \ - indexQueue->enqueue(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); \ - IndexGenerator::getInstance()->creatAllIndex(indexQueue); \ - indexQueue->clear(); - -#define CREATE_FILE_CONTENT_INDEX \ - if ((!QString(event->name).split(".").isEmpty()) && (true == this->targetFileTypeMap[QString(event->name).split(".").last()])) { \ - contentIndexQueue->enqueue(QString(currentPath[event->wd] + '/' + event->name)); \ - IndexGenerator::getInstance()->creatAllIndex(contentIndexQueue); \ - contentIndexQueue->clear(); \ - break; \ - } - -#define TRAVERSE_DIR \ - QString tmp = currentPath[event->wd] + '/' + event->name; \ - QFileInfo fi(tmp); \ - if(!fi.isSymLink()){ \ - AddWatch(tmp); \ - setPath(tmp); \ - Traverse(); \ - } - - -#define CREATE_FILE \ - CREATE_FILE_NAME_INDEX \ - CREATE_FILE_CONTENT_INDEX -using namespace Zeeker; -InotifyIndex::InotifyIndex(const QString& path) : Traverse_BFS(path) { - qDebug() << "setInotifyMaxUserWatches start"; - UkuiSearchQDBus usQDBus; - usQDBus.setInotifyMaxUserWatches(); - qDebug() << "setInotifyMaxUserWatches end"; - m_sharedMemory = new QSharedMemory("ukui-search-shared-map", this); -} - -InotifyIndex::~InotifyIndex() { - IndexGenerator::getInstance()->~IndexGenerator(); - qWarning() << "~InotifyIndex"; -} - -void InotifyIndex::firstTraverse() { - QQueue bfs; - bfs.enqueue(this->path); - QFileInfoList list; - QDir dir; - dir.setFilter(QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot); - dir.setSorting(QDir::DirsFirst); - while(!bfs.empty()) { - dir.setPath(bfs.dequeue()); - list = dir.entryInfoList(); - for(auto i : list) { - if(i.isDir() && (!(i.isSymLink()))) { - this->AddWatch(i.absoluteFilePath()); - bfs.enqueue(i.absoluteFilePath()); - } - } - } -} - -void InotifyIndex::DoSomething(const QFileInfo& fileInfo) { - qDebug() << fileInfo.fileName() << "-------" << fileInfo.absoluteFilePath(); - if(fileInfo.isDir() && (!fileInfo.isSymLink())) { - this->AddWatch(fileInfo.absoluteFilePath()); - } - QQueue > tempFile; - tempFile.enqueue(QVector() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString((fileInfo.isDir() && (!fileInfo.isSymLink())) ? "1" : "0")); - IndexGenerator::getInstance()->creatAllIndex(&tempFile); - if((fileInfo.fileName().split(".", QString::SkipEmptyParts).length() > 1) && (true == targetFileTypeMap[fileInfo.fileName().split(".").last()])) { - QQueue tmp; - tmp.enqueue(fileInfo.absoluteFilePath()); - IndexGenerator::getInstance()->creatAllIndex(&tmp); - } -} - -bool InotifyIndex::AddWatch(const QString &path) { - int ret = inotify_add_watch(m_fd, path.toStdString().c_str(), (IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | IN_DELETE | IN_MODIFY)); - if(ret == -1) { - qDebug() << "AddWatch error:" << path; - return false; - } -// Q_ASSERT(ret != -1); - assert(ret != -1); - currentPath[ret] = path; -// qDebug() << "Watch: " << path << "ret: " << ret; - return true; -} - -bool InotifyIndex::RemoveWatch(const QString &path, bool removeFromDatabase) { - int ret = inotify_rm_watch(m_fd, currentPath.key(path)); - if(ret) { - qDebug() << "remove path error"; - return false; - } -// Q_ASSERT(ret == 0); - assert(ret == 0); - - if(removeFromDatabase) { - for(QMap::Iterator i = currentPath.begin(); i != currentPath.end();) { - // qDebug() << i.value(); - if(i.value().length() > path.length()) { - // if (i.value().mid(0, path.length()) == path){ - // if (path.startsWith(i.value())){ - if(i.value().startsWith(path)) { - qDebug() << "remove path: " << i.value(); - ret = inotify_rm_watch(m_fd, currentPath.key(path)); - if(ret) { - qDebug() << "remove path error"; - // return false; - } - // assert(ret == 0); - /*--------------------------------*/ - //在此调用删除索引 - qDebug() << i.value(); - IndexGenerator::getInstance()->deleteAllIndex(new QStringList(i.value())); - /*--------------------------------*/ - currentPath.erase(i++); - // i++; - } else { - i++; - } - } else { - i++; - } - } - } else { - for(QMap::Iterator i = currentPath.begin(); i != currentPath.end();) { - // qDebug() << i.value(); - if(i.value().length() > path.length()) { - // if (i.value().mid(0, path.length()) == path){ - // if (path.startsWith(i.value())){ - if(i.value().startsWith(path)) { - qDebug() << "remove path: " << i.value(); - ret = inotify_rm_watch(m_fd, currentPath.key(path)); - if(ret) { - qDebug() << "remove path error"; - // return false; - } - // assert(ret == 0); - currentPath.erase(i++); - // i++; - } else { - i++; - } - } else { - i++; - } - } - } -// qDebug() << path; - //这个貌似不用删，先mark一下 -// currentPath.remove(currentPath.key(path)); - return true; -} - -void InotifyIndex::eventProcess(const char* buf, ssize_t tmp) { - QQueue>* indexQueue = new QQueue>(); - QQueue* contentIndexQueue = new QQueue(); - - ssize_t numRead = 0; - numRead = tmp; - char * p = const_cast(buf); - IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "0"); - for(; p < buf + numRead;) { - struct inotify_event * event = reinterpret_cast(p); - if(event->name[0] != '.') { - - qDebug() << "Read Event event->wd: " << event->wd; - qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask; - - qDebug() << QString(currentPath[event->wd] + '/' + event->name); - // switch (event->mask) { - if(event->mask & IN_CREATE) { - - //Create top dir first, traverse it last. - qDebug() << "IN_CREATE"; - CREATE_FILE - if(event->mask & IN_ISDIR) { - TRAVERSE_DIR - } - goto next; - } - - if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)) { - qDebug() << "IN_DELETE or IN_MOVED_FROM"; - if(event->mask & IN_ISDIR) { - RemoveWatch(currentPath[event->wd] + '/' + event->name); - } - //delete once more - IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); - goto next; - } - - if(event->mask & IN_MODIFY) { - qDebug() << "IN_MODIFY"; - if(!(event->mask & IN_ISDIR)) { -// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); - CREATE_FILE - } - goto next; - } - - if(event->mask & IN_MOVED_TO) { - qDebug() << "IN_MOVED_TO"; - if(event->mask & IN_ISDIR) { - RemoveWatch(currentPath[event->wd] + '/' + event->name); -// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); - CREATE_FILE - TRAVERSE_DIR - } else { - IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); - CREATE_FILE - } - goto next; - } - } -next: - p += sizeof(struct inotify_event) + event->len; - } - IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2"); - delete indexQueue; - indexQueue = nullptr; - delete contentIndexQueue; - contentIndexQueue = nullptr; -} - -void InotifyIndex::run() { - m_fd = inotify_init(); - qDebug() << "m_fd----------->" << m_fd; - - this->AddWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation)); - this->setPath(QStandardPaths::writableLocation(QStandardPaths::HomeLocation)); - this->firstTraverse(); - - int fifo_fd; - char buffer[2]; - memset(buffer, 0, sizeof(buffer)); - fifo_fd = open(UKUI_SEARCH_PIPE_PATH, O_RDWR); - if(fifo_fd == -1) { - perror("open fifo error\n"); - assert(false); - } - int retval = read(fifo_fd, buffer, sizeof(buffer)); - if(retval == -1) { - perror("read error\n"); - assert(false); - } - printf("read fifo=[%s]\n", buffer); - - printf("read data ok\n"); - close(fifo_fd); - if(buffer[0] & 0x1) { - printf("data confirmed\n"); - } - unlink(UKUI_SEARCH_PIPE_PATH); - - char buf[BUF_LEN] __attribute__((aligned(8))); - - ssize_t numRead; - - while(FileUtils::SearchMethod::INDEXSEARCH == FileUtils::searchMethod) { -// for (;;) { /* Read events forever */ - memset(buf, 0x00, BUF_LEN); - numRead = read(m_fd, buf, BUF_LEN); - - if(numRead == -1) { - printf("\033[1;31;40mread event error\033[0m\n"); - IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1"); - fflush(stdout); - assert(false); - } - - //TODO: Merge multiple signals. - char * tmp = const_cast(buf); - - for(; tmp < buf + numRead;) { - struct inotify_event * event = reinterpret_cast(tmp); - if(event->name[0] != '.') { -// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask; -// qDebug("mask:0x%x,",event->mask); - break; - } - tmp += sizeof(struct inotify_event) + event->len; - } - if(tmp >= buf + numRead) { - continue; - } - - ++FileUtils::_index_status; - - pid_t pid; - pid = fork(); - if(pid == 0) { - prctl(PR_SET_PDEATHSIG, SIGTERM); - prctl(PR_SET_NAME, "inotify-index"); - if(numRead == 0) { - qDebug() << "read() from inotify fd returned 0!"; - } - if(numRead == -1) { - qDebug() << "read"; - } - eventProcess(buf, numRead); - - fd_set read_fds; - int rc; - timeval* read_timeout = (timeval*)malloc(sizeof(timeval)); - - read_timeout->tv_sec = 40; - read_timeout->tv_usec = 0; - for(;;) { - FD_ZERO(&read_fds); - FD_SET(m_fd, &read_fds); - qDebug() << read_timeout->tv_sec; - rc = select(m_fd + 1, &read_fds, NULL, NULL, read_timeout); - if(rc < 0) { - // error - qWarning() << "select result < 0, error!"; - IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1"); - assert(false); - } else if(rc == 0) { - qDebug() << "select timeout!"; - ::free(read_timeout); - IndexGenerator::getInstance()->~IndexGenerator(); - QBuffer buffer; - QDataStream out(&buffer); - if (m_sharedMemory->isAttached()) { - m_sharedMemory->detach(); - } - buffer.open(QBuffer::ReadWrite); - out << currentPath; - int size = buffer.size(); - if (!m_sharedMemory->create(size)) { - qDebug() << "Create sharedMemory Error: " << m_sharedMemory->errorString(); - } else { - m_sharedMemory->lock(); - char *to = static_cast(m_sharedMemory->data()); - const char *from = buffer.data().constData(); - memcpy(to, from, qMin(size, m_sharedMemory->size())); - m_sharedMemory->unlock(); - } - // GlobalSettings::getInstance()->forceSync(); - ::_exit(0); - } else { - memset(buf, 0x00, BUF_LEN); - numRead = read(m_fd, buf, BUF_LEN); - if(numRead == -1) { - printf("\033[1;31;40mread event error\033[0m\n"); - fflush(stdout); - assert(false); - } - - //TODO: Merge multiple signals. - char * tmp = const_cast(buf); - - for(; tmp < buf + numRead; ) { - struct inotify_event * event = reinterpret_cast(tmp); - if(event->name[0] != '.') { - break; - } - tmp += sizeof(struct inotify_event) + event->len; - } - if(tmp >= buf + numRead) { - continue; - } - - qDebug() << "Read " << numRead << " bytes from inotify fd"; - this->eventProcess(buf, numRead); - } - } - } else if(pid > 0) { - memset(buf, 0x00, BUF_LEN); - waitpid(pid, NULL, 0); - if (!m_sharedMemory->attach()) { - qDebug() << "SharedMemory attach Error: " << m_sharedMemory->errorString(); - } else { - QBuffer buffer; - QDataStream in(&buffer); - QMap pathMap; - m_sharedMemory->lock(); - buffer.setData(static_cast(m_sharedMemory->constData()), m_sharedMemory->size()); - buffer.open(QBuffer::ReadWrite); - in >> pathMap; - m_sharedMemory->unlock(); - m_sharedMemory->detach(); - currentPath = pathMap; - } - --FileUtils::_index_status; - } else { - assert(false); - } - } - - if(FileUtils::SearchMethod::DIRECTSEARCH == FileUtils::searchMethod) { - IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "3"); - RemoveWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation), false); - } -} diff --git a/libsearch/index/inotify-index.h b/libsearch/index/inotify-index.h deleted file mode 100644 index 1b31f2a..0000000 --- a/libsearch/index/inotify-index.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (C) 2020, KylinSoft Co., Ltd. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * Authors: zhangzihao - * - */ -#ifndef INOTIFYINDEX_H -#define INOTIFYINDEX_H - -#include -#include -#include -#include -#include -#include "index-generator.h" -#include "traverse_bfs.h" -#include "ukui-search-qdbus.h" -#include "index-status-recorder.h" -#include "file-utils.h" -#include "first-index.h" -#include "common.h" - -#define BUF_LEN 1024000 - -namespace Zeeker { -class InotifyIndex; -static InotifyIndex* global_instance_of_index = nullptr; -class InotifyIndex : public QThread, public Traverse_BFS { - Q_OBJECT -public: - static InotifyIndex* getInstance(const QString& path) { - if(!global_instance_of_index) { - global_instance_of_index = new InotifyIndex(path); - } - return global_instance_of_index; - } - InotifyIndex(const QString&); - ~InotifyIndex(); - - bool AddWatch(const QString&); - bool RemoveWatch(const QString&, bool removeFromDatabase = true); - virtual void DoSomething(const QFileInfo &) final; - - void eventProcess(const char*, ssize_t); - void firstTraverse(); -protected: - void run() override; -private: - int m_fd; - - QMap currentPath; - - const QMap targetFileTypeMap = { - std::map::value_type("doc", true), - std::map::value_type("docx", true), - std::map::value_type("ppt", true), - std::map::value_type("pptx", true), - std::map::value_type("xls", true), - std::map::value_type("xlsx", true), - std::map::value_type("txt", true), - std::map::value_type("dot", true), - std::map::value_type("wps", true), - std::map::value_type("pps", true), - std::map::value_type("dps", true), - std::map::value_type("et", true), - std::map::value_type("pdf", true) - }; - QSharedMemory *m_sharedMemory = nullptr; - -}; -} - -#endif // INOTIFYINDEX_H diff --git a/libsearch/index/inotify-watch.cpp b/libsearch/index/inotify-watch.cpp index b36e705..3d1d8e3 100644 --- a/libsearch/index/inotify-watch.cpp +++ b/libsearch/index/inotify-watch.cpp @@ -180,10 +180,11 @@ void InotifyWatch::run() int rc; rc = select(m_inotifyFd + 1, &fds, NULL, NULL, NULL); if(rc > 0) { - ++FileUtils::_index_status; + ++FileUtils::indexStatus; int avail; if (ioctl(m_inotifyFd, FIONREAD, &avail) == EINVAL) { qWarning() << "Did not receive an entire inotify event."; + --FileUtils::indexStatus; return; } @@ -211,7 +212,7 @@ void InotifyWatch::run() slotEvent(buf, len); free(buf); } - --FileUtils::_index_status; + --FileUtils::indexStatus; } else if(rc < 0) { // error qWarning() << "select result < 0, error!"; diff --git a/libsearch/index/searchmethodmanager.cpp b/libsearch/index/searchmethodmanager.cpp index bfdc7ee..b5b8ec5 100644 --- a/libsearch/index/searchmethodmanager.cpp +++ b/libsearch/index/searchmethodmanager.cpp @@ -12,7 +12,7 @@ void SearchMethodManager::searchMethod(FileUtils::SearchMethod sm) { } else { qWarning("enum class error!!!\n"); } - if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::_index_status) { + if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::indexStatus) { // Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running. QDir fifoDir = QDir(QDir::homePath() + "/.config/org.ukui/ukui-search"); diff --git a/src/main.cpp b/src/main.cpp index 04bdb83..faec524 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -131,7 +131,7 @@ void searchMethod(FileUtils::SearchMethod sm){ printf("enum class error!!!\n"); qWarning("enum class error!!!\n"); } - if (FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::_index_status) { + if (FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::indexStatus) { qWarning() << "start first index"; FirstIndex fi("/home/zhangzihao/Desktop"); fi.start(); diff --git a/src/settings-widget.cpp b/src/settings-widget.cpp index ca98ca2..1d82665 100644 --- a/src/settings-widget.cpp +++ b/src/settings-widget.cpp @@ -272,8 +272,8 @@ void SettingsWidget::clearLayout(QLayout * layout) { * @brief SettingsWidget::refreshIndexState 定时刷新索引项 */ void SettingsWidget::refreshIndexState() { -// qDebug()<<"FileUtils::_index_status: "<setIndexState(true); } else { this->setIndexState(false); @@ -281,8 +281,8 @@ void SettingsWidget::refreshIndexState() { m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::_max_index_count))); m_timer = new QTimer; connect(m_timer, &QTimer::timeout, this, [ = ]() { - qDebug() << "FileUtils::_index_status: " << FileUtils::_index_status; - if(FileUtils::_index_status != 0) { + qDebug() << "FileUtils::indexStatus: " << FileUtils::indexStatus; + if(FileUtils::indexStatus != 0) { this->setIndexState(true); } else { this->setIndexState(false);