From b9da7ad70826644c5fa849e63059c6b4adebfb24 Mon Sep 17 00:00:00 2001 From: iaom <18504285112@163.com> Date: Thu, 10 Jun 2021 20:43:57 +0800 Subject: [PATCH] Add inotify events queue for merging events. --- libsearch/common.h | 19 +- libsearch/file-utils.cpp | 2 +- libsearch/index/construct-document.cpp | 21 +- libsearch/index/index-generator.cpp | 166 +++----- libsearch/index/index-generator.h | 19 +- libsearch/index/index-status-recorder.cpp | 2 + libsearch/index/index-status-recorder.h | 3 + libsearch/index/index.pri | 6 + libsearch/index/inotify-watch.cpp | 443 ++++++++++++++++++++++ libsearch/index/inotify-watch.h | 52 +++ libsearch/index/pending-file-queue.cpp | 151 ++++++++ libsearch/index/pending-file-queue.h | 65 ++++ libsearch/index/pending-file.cpp | 81 ++++ libsearch/index/pending-file.h | 65 ++++ libsearch/index/searchmethodmanager.cpp | 19 +- libsearch/index/searchmethodmanager.h | 8 +- 16 files changed, 992 insertions(+), 130 deletions(-) create mode 100644 libsearch/index/inotify-watch.cpp create mode 100644 libsearch/index/inotify-watch.h create mode 100644 libsearch/index/pending-file-queue.cpp create mode 100644 libsearch/index/pending-file-queue.h create mode 100644 libsearch/index/pending-file.cpp create mode 100644 libsearch/index/pending-file.h diff --git a/libsearch/common.h b/libsearch/common.h index d3c238c..59f0239 100644 --- a/libsearch/common.h +++ b/libsearch/common.h @@ -1,7 +1,22 @@ #ifndef COMMON_H #define COMMON_H - +#include #define UKUI_SEARCH_PIPE_PATH (QDir::homePath()+"/.config/org.ukui/ukui-search/ukuisearch").toLocal8Bit().constData() - +#define HOME_PATH QDir::homePath() +static const QMap targetFileTypeMap = { + std::map::value_type("doc", true), + std::map::value_type("docx", true), + std::map::value_type("ppt", true), + std::map::value_type("pptx", true), + std::map::value_type("xls", true), + std::map::value_type("xlsx", true), + std::map::value_type("txt", true), + std::map::value_type("dot", true), + std::map::value_type("wps", true), + std::map::value_type("pps", true), + std::map::value_type("dps", true), + std::map::value_type("et", true), + std::map::value_type("pdf", true) +}; //TODO Put things that needed to be put here here. #endif // COMMON_H diff --git a/libsearch/file-utils.cpp b/libsearch/file-utils.cpp index 732be89..c6fdacd 100644 --- a/libsearch/file-utils.cpp +++ b/libsearch/file-utils.cpp @@ -516,7 +516,7 @@ void FileUtils::getDocxTextContent(QString &path, QString &textcontent) { QDomElement wr = wp.firstChildElement("w:r"); while(!wr.isNull()) { QDomElement wt = wr.firstChildElement("w:t"); - textcontent.append(wt.text().replace("\n", "")); + textcontent.append(wt.text().replace("\n", "")).replace("\r", " "); if(textcontent.length() >= MAX_CONTENT_LENGTH / 3) { file.close(); return; diff --git a/libsearch/index/construct-document.cpp b/libsearch/index/construct-document.cpp index e54ff84..4d3a318 100644 --- a/libsearch/index/construct-document.cpp +++ b/libsearch/index/construct-document.cpp @@ -34,8 +34,8 @@ ConstructDocumentForPath::ConstructDocumentForPath(QVector list) { void ConstructDocumentForPath::run() { // qDebug()<<"ConstructDocumentForPath"; - if(!Zeeker::_doc_list_path) - Zeeker::_doc_list_path = new QList; +// if(!Zeeker::_doc_list_path) +// Zeeker::_doc_list_path = new QVector; // qDebug()<<_doc_list_path->size(); QString index_text = m_list.at(0).toLower(); QString sourcePath = m_list.at(1); @@ -87,9 +87,9 @@ void ConstructDocumentForPath::run() { } // QMetaObject::invokeMethod(m_indexGenerator,"appendDocListPath",Q_ARG(Document,doc)); - Zeeker::_mutex_doc_list_path.lock(); - Zeeker::_doc_list_path->append(doc); - Zeeker::_mutex_doc_list_path.unlock(); + IndexGenerator::_mutex_doc_list_path.lock(); + IndexGenerator::_doc_list_path.append(doc); + IndexGenerator::_mutex_doc_list_path.unlock(); // qDebug()<<"ConstructDocumentForPath finish"; return; } @@ -102,8 +102,8 @@ ConstructDocumentForContent::ConstructDocumentForContent(QString path) { void ConstructDocumentForContent::run() { // qDebug() << "ConstructDocumentForContent currentThreadId()" << QThread::currentThreadId(); // 构造文本索引的document - if(!Zeeker::_doc_list_content) - Zeeker::_doc_list_content = new QList; +// if(!Zeeker::_doc_list_content) +// Zeeker::_doc_list_content = new QVector; QString content; FileReader::getTextContent(m_path, content); if(content.isEmpty()) @@ -127,11 +127,12 @@ void ConstructDocumentForContent::run() { doc.addPosting(term.at(i).word, term.at(i).offsets, static_cast(term.at(i).weight)); } - Zeeker::_mutex_doc_list_content.lock(); - Zeeker::_doc_list_content->append(doc); - Zeeker::_mutex_doc_list_content.unlock(); + IndexGenerator::_mutex_doc_list_content.lock(); + IndexGenerator::_doc_list_content.append(doc); + IndexGenerator::_mutex_doc_list_content.unlock(); content.clear(); content.squeeze(); term.clear(); + term.shrink_to_fit(); return; } diff --git a/libsearch/index/index-generator.cpp b/libsearch/index/index-generator.cpp index 8c02122..896852f 100644 --- a/libsearch/index/index-generator.cpp +++ b/libsearch/index/index-generator.cpp @@ -28,7 +28,6 @@ #include "file-utils.h" #include "index-generator.h" #include "chinese-segmentation.h" -#include "construct-document.h" #include @@ -39,10 +38,14 @@ using namespace Zeeker; static IndexGenerator *global_instance = nullptr; QMutex IndexGenerator::m_mutex; -QList *Zeeker::_doc_list_path; -QMutex Zeeker::_mutex_doc_list_path; -QList *Zeeker::_doc_list_content; -QMutex Zeeker::_mutex_doc_list_content; +//QVector *Zeeker::_doc_list_path; +//QMutex Zeeker::_mutex_doc_list_path; +//QVector *Zeeker::_doc_list_content; +//QMutex Zeeker::_mutex_doc_list_content; +QMutex IndexGenerator::_mutex_doc_list_path; +QMutex IndexGenerator::_mutex_doc_list_content; +QVector IndexGenerator::_doc_list_path = QVector(); +QVector IndexGenerator::_doc_list_content = QVector(); IndexGenerator *IndexGenerator::getInstance(bool rebuild, QObject *parent) { QMutexLocker locker(&m_mutex); @@ -61,45 +64,33 @@ bool IndexGenerator::setIndexdataPath() { //文件名索引 bool IndexGenerator::creatAllIndex(QQueue > *messageList) { -// FileUtils::_index_status |= 0x1; -// qDebug() << messageList->size(); HandlePathList(messageList); - if(_doc_list_path == NULL) { +// if(_doc_list_path == NULL) { +// return false; +// } + if(IndexGenerator::_doc_list_path.isEmpty()) { return false; } qDebug() << "begin creatAllIndex"; -// GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "0"); try { -// m_indexer = new Xapian::TermGenerator(); -// m_indexer.set_database(*m_database_path); - //可以实现拼写纠正 -// m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING); -// m_indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME); - -// int count =0; - - for(auto i : *_doc_list_path) { + for(auto i : IndexGenerator::_doc_list_path) { insertIntoDatabase(i); -// if(++count > 8999){ -// count = 0; -// m_database_path->commit(); -// } } m_database_path->commit(); } catch(const Xapian::Error &e) { qWarning() << "creatAllIndex fail!" << QString::fromStdString(e.get_description()); //need a record - IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1"); -// FileUtils::_index_status &= ~0x1; + IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1"); assert(false); } -// GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2"); qDebug() << "finish creatAllIndex"; -// FileUtils::_index_status &= ~0x1; - _doc_list_path->clear(); - delete _doc_list_path; - _doc_list_path = nullptr; + IndexGenerator::_doc_list_path.clear(); + IndexGenerator::_doc_list_path.squeeze(); + QVector().swap(IndexGenerator::_doc_list_path); + +// delete _doc_list_path; +// _doc_list_path = nullptr; return true; } //文件内容索引 @@ -107,16 +98,19 @@ bool IndexGenerator::creatAllIndex(QQueue *messageList) { // FileUtils::_index_status |= 0x2; HandlePathList(messageList); qDebug() << "begin creatAllIndex for content"; - if(_doc_list_content == NULL) { +// if(_doc_list_content == NULL) { +// return false; +// } + if(IndexGenerator::_doc_list_content.isEmpty()) { return false; } - int size = _doc_list_content->size(); + int size = IndexGenerator::_doc_list_content.size(); qDebug() << "begin creatAllIndex for content" << size; if(!size == 0) { // GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "0"); try { int count = 0; - for(auto i : *_doc_list_content) { + for(auto i : IndexGenerator::_doc_list_content) { insertIntoContentDatabase(i); if(++count > 999) { count = 0; @@ -133,9 +127,11 @@ bool IndexGenerator::creatAllIndex(QQueue *messageList) { // GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2"); // FileUtils::_index_status &= ~0x2; qDebug() << "finish creatAllIndex for content"; - _doc_list_content->clear(); - delete _doc_list_content; - _doc_list_content = nullptr; + IndexGenerator::_doc_list_content.clear(); + IndexGenerator::_doc_list_content.squeeze(); + QVector().swap(IndexGenerator::_doc_list_content); +// delete _doc_list_content; +// _doc_list_content = nullptr; } Q_EMIT this->transactionFinished(); return true; @@ -297,7 +293,7 @@ void IndexGenerator::HandlePathList(QQueue *messageList) { return; } - +//deprecated Document IndexGenerator::GenerateDocument(const QVector &list) { Document doc; // qDebug()< &list) { return doc; } - +//deprecated Document IndexGenerator::GenerateContentDocument(const QString &path) { // 构造文本索引的document QString content; @@ -389,7 +385,7 @@ bool IndexGenerator::isIndexdataExist() { } - +//deprecated QStringList IndexGenerator::IndexSearch(QString indexText) { QStringList searchResult; try { @@ -455,71 +451,6 @@ QStringList IndexGenerator::IndexSearch(QString indexText) { return searchResult; } -//void IndexGenerator::setSynonym() -//{ -// try -// { -// m_database_path->add_synonym("a","A"); -// m_database_path->add_synonym("b","B"); -// m_database_path->add_synonym("c","C"); -// m_database_path->add_synonym("d","D"); -// m_database_path->add_synonym("e","A"); -// m_database_path->add_synonym("f","F"); -// m_database_path->add_synonym("g","G"); -// m_database_path->add_synonym("h","H"); -// m_database_path->add_synonym("i","I"); -// m_database_path->add_synonym("j","J"); -// m_database_path->add_synonym("k","K"); -// m_database_path->add_synonym("l","L"); -// m_database_path->add_synonym("m","M"); -// m_database_path->add_synonym("n","N"); -// m_database_path->add_synonym("o","O"); -// m_database_path->add_synonym("p","P"); -// m_database_path->add_synonym("q","Q"); -// m_database_path->add_synonym("r","R"); -// m_database_path->add_synonym("s","S"); -// m_database_path->add_synonym("t","T"); -// m_database_path->add_synonym("u","U"); -// m_database_path->add_synonym("v","V"); -// m_database_path->add_synonym("w","W"); -// m_database_path->add_synonym("x","X"); -// m_database_path->add_synonym("y","Y"); -// m_database_path->add_synonym("z","Z"); - -// m_database_path->add_synonym("A","a"); -// m_database_path->add_synonym("B","b"); -// m_database_path->add_synonym("C","c"); -// m_database_path->add_synonym("D","d"); -// m_database_path->add_synonym("E","e"); -// m_database_path->add_synonym("F","f"); -// m_database_path->add_synonym("G","g"); -// m_database_path->add_synonym("H","h"); -// m_database_path->add_synonym("I","i"); -// m_database_path->add_synonym("J","j"); -// m_database_path->add_synonym("K","k"); -// m_database_path->add_synonym("L","a"); -// m_database_path->add_synonym("M","m"); -// m_database_path->add_synonym("N","n"); -// m_database_path->add_synonym("O","o"); -// m_database_path->add_synonym("P","p"); -// m_database_path->add_synonym("Q","q"); -// m_database_path->add_synonym("R","r"); -// m_database_path->add_synonym("S","s"); -// m_database_path->add_synonym("T","t"); -// m_database_path->add_synonym("U","u"); -// m_database_path->add_synonym("V","v"); -// m_database_path->add_synonym("W","w"); -// m_database_path->add_synonym("X","x"); -// m_database_path->add_synonym("Y","y"); -// m_database_path->add_synonym("Z","z"); -// m_database_path->commit(); -// } -// catch(const Xapian::Error &e) -// { -// qWarning() <isEmpty()) @@ -548,3 +479,32 @@ bool IndexGenerator::deleteAllIndex(QStringList *pathlist) { return true; } +bool IndexGenerator::updateIndex(QVector *pendingFiles) +{ + QQueue> *fileIndexInfo = new QQueue>; + QQueue *fileContentIndexInfo = new QQueue; + QStringList *deleteList = new QStringList; + for(PendingFile file : *pendingFiles) { + if(file.shouldRemoveIndex()) { + + deleteList->append(file.path()); + continue; + } + fileIndexInfo->append(QVector() << file.path().section("/" , -1) << file.path() << QString(file.isDir() ? "1" : "0")); + if((!file.path().split(".").isEmpty()) && (true == targetFileTypeMap[file.path().section("/" , -1) .split(".").last()])) + fileContentIndexInfo->append(file.path()); + } + if(!deleteList->isEmpty()) { + deleteAllIndex(deleteList); + } + if(!fileIndexInfo->isEmpty()) { + creatAllIndex(fileIndexInfo); + } + if(!fileContentIndexInfo->isEmpty()) { + creatAllIndex(fileContentIndexInfo); + } + delete fileIndexInfo; + delete fileContentIndexInfo; + return true; +} + diff --git a/libsearch/index/index-generator.h b/libsearch/index/index-generator.h index bc614b2..873a6cd 100644 --- a/libsearch/index/index-generator.h +++ b/libsearch/index/index-generator.h @@ -29,18 +29,22 @@ #include #include //#include +#include "construct-document.h" #include "index-status-recorder.h" #include "document.h" #include "file-reader.h" #include "common.h" +#include "pending-file.h" namespace Zeeker { -extern QList *_doc_list_path; -extern QMutex _mutex_doc_list_path; -extern QList *_doc_list_content; -extern QMutex _mutex_doc_list_content; +//extern QVector *_doc_list_path; +//extern QMutex _mutex_doc_list_path; +//extern QVector *_doc_list_content; +//extern QMutex _mutex_doc_list_content; class IndexGenerator : public QObject { + friend class ConstructDocumentForPath; + friend class ConstructDocumentForContent; Q_OBJECT public: static IndexGenerator *getInstance(bool rebuild = false, QObject *parent = nullptr); @@ -58,6 +62,7 @@ public Q_SLOTS: bool creatAllIndex(QQueue> *messageList); bool creatAllIndex(QQueue *messageList); bool deleteAllIndex(QStringList *pathlist); + bool updateIndex(QVector *pendingFiles); private: explicit IndexGenerator(bool rebuild = false, QObject *parent = nullptr); @@ -72,8 +77,10 @@ private: void insertIntoDatabase(Document& doc); void insertIntoContentDatabase(Document& doc); -// QList *m_doc_list_path; //for path index -// QList *m_doc_list_content; // for text content index + static QVector _doc_list_path; + static QMutex _mutex_doc_list_path; + static QVector _doc_list_content; + static QMutex _mutex_doc_list_content; QMap m_index_map; QString m_index_data_path; Xapian::WritableDatabase* m_database_path; diff --git a/libsearch/index/index-status-recorder.cpp b/libsearch/index/index-status-recorder.cpp index 63cba32..9089722 100644 --- a/libsearch/index/index-status-recorder.cpp +++ b/libsearch/index/index-status-recorder.cpp @@ -13,8 +13,10 @@ IndexStatusRecorder *IndexStatusRecorder::getInstance() void IndexStatusRecorder::setStatus(const QString &key, const QVariant &value) { + m_mutex.lock(); m_status->setValue(key, value); m_status->sync(); + m_mutex.unlock(); } const QVariant IndexStatusRecorder::getStatus(const QString &key) diff --git a/libsearch/index/index-status-recorder.h b/libsearch/index/index-status-recorder.h index 25f8009..bf65850 100644 --- a/libsearch/index/index-status-recorder.h +++ b/libsearch/index/index-status-recorder.h @@ -4,9 +4,11 @@ #include #include #include +#include #define CONTENT_INDEX_DATABASE_STATE "content_index_database_state" #define INDEX_DATABASE_STATE "index_database_state" #define INOTIFY_NORMAL_EXIT "inotify_normal_exit" +#define PENDING_FILE_QUEUE_FINISH "pending_file_queue_finish" #define INDEX_STATUS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-index-status.conf" namespace Zeeker { //fixme: we need a better way to record index status. @@ -21,6 +23,7 @@ public: private: explicit IndexStatusRecorder(QObject *parent = nullptr); QSettings *m_status; + QMutex m_mutex; }; } diff --git a/libsearch/index/index.pri b/libsearch/index/index.pri index 6ceadc6..ddb98c5 100644 --- a/libsearch/index/index.pri +++ b/libsearch/index/index.pri @@ -8,6 +8,9 @@ HEADERS += \ $$PWD/index-generator.h \ $$PWD/index-status-recorder.h \ $$PWD/inotify-index.h \ + $$PWD/inotify-watch.h \ + $$PWD/pending-file-queue.h \ + $$PWD/pending-file.h \ $$PWD/search-manager.h \ $$PWD/searchmethodmanager.h \ $$PWD/traverse_bfs.h \ @@ -21,6 +24,9 @@ SOURCES += \ $$PWD/index-generator.cpp \ $$PWD/index-status-recorder.cpp \ $$PWD/inotify-index.cpp \ + $$PWD/inotify-watch.cpp \ + $$PWD/pending-file-queue.cpp \ + $$PWD/pending-file.cpp \ $$PWD/search-manager.cpp \ $$PWD/searchmethodmanager.cpp \ $$PWD/traverse_bfs.cpp \ diff --git a/libsearch/index/inotify-watch.cpp b/libsearch/index/inotify-watch.cpp new file mode 100644 index 0000000..ba327b6 --- /dev/null +++ b/libsearch/index/inotify-watch.cpp @@ -0,0 +1,443 @@ +#include "inotify-watch.h" +#include +#include +using namespace Zeeker; +static InotifyWatch* global_instance_InotifyWatch = nullptr; + +Zeeker::InotifyWatch *Zeeker::InotifyWatch::getInstance(const QString &path) +{ + if(!global_instance_InotifyWatch) { + global_instance_InotifyWatch = new InotifyWatch(path); + } + return global_instance_InotifyWatch; +} + +Zeeker::InotifyWatch::InotifyWatch(const QString &path): Traverse_BFS(path) +{ + qDebug() << "setInotifyMaxUserWatches start"; + UkuiSearchQDBus usQDBus; + usQDBus.setInotifyMaxUserWatches(); + qDebug() << "setInotifyMaxUserWatches end"; + m_sharedMemory = new QSharedMemory("ukui-search-shared-map", this); +} + +InotifyWatch::~InotifyWatch() +{ + if(m_notifier) + delete m_notifier; + m_notifier = nullptr; +} + +bool InotifyWatch::addWatch(const QString &path) +{ + int ret = inotify_add_watch(m_inotifyFd, path.toStdString().c_str(), (IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | IN_DELETE | IN_MODIFY)); + if(ret == -1) { + qWarning() << "AddWatch error:" << path; + return false; + } + currentPath[ret] = path; +// qDebug() << "Watch: " << path << "ret: " << ret; + return true; +} + +bool InotifyWatch::removeWatch(const QString &path, bool removeFromDatabase) +{ + inotify_rm_watch(m_inotifyFd, currentPath.key(path)); + + if(removeFromDatabase) { + for(QMap::Iterator i = currentPath.begin(); i != currentPath.end();) { + // qDebug() << i.value(); + // if(i.value().length() > path.length()) { + if(i.value().startsWith(path)) { + qDebug() << "remove path: " << i.value(); + inotify_rm_watch(m_inotifyFd, currentPath.key(path)); + PendingFile f(i.value()); + f.setDeleted(); + f.setIsDir(); + PendingFileQueue::getInstance()->enqueue(f); + currentPath.erase(i++); + } else { + i++; + } + } + } else { + for(QMap::Iterator i = currentPath.begin(); i != currentPath.end();) { + // qDebug() << i.value(); + if(i.value().length() > path.length()) { + if(i.value().startsWith(path)) { + qDebug() << "remove path: " << i.value(); + inotify_rm_watch(m_inotifyFd, currentPath.key(path)); + currentPath.erase(i++); + } else { + i++; + } + } else { + i++; + } + } + } + currentPath.remove(currentPath.key(path)); + return true; +} + +void InotifyWatch::DoSomething(const QFileInfo &info) +{ + qDebug() << info.fileName() << "-------" << info.absoluteFilePath(); + if(info.isDir() && (!info.isSymLink())) { + this->addWatch(info.absoluteFilePath()); + } + PendingFile f(info.absoluteFilePath()); + if(info.isDir()) { + f.setIsDir(); + } + PendingFileQueue::getInstance()->enqueue(f); +} + +void InotifyWatch::firstTraverse() +{ + QQueue bfs; + bfs.enqueue(this->path); + QFileInfoList list; + QDir dir; + dir.setFilter(QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot); + dir.setSorting(QDir::DirsFirst); + while(!bfs.empty()) { + dir.setPath(bfs.dequeue()); + list = dir.entryInfoList(); + for(auto i : list) { + if(i.isDir() && (!(i.isSymLink()))) { + this->addWatch(i.absoluteFilePath()); + bfs.enqueue(i.absoluteFilePath()); + } + } + } +} + +void InotifyWatch::stopWatch() +{ +// if(this->isRunning()) { +// this->quit(); +// if(m_notifier) +// delete m_notifier; +// m_notifier = nullptr; +// removeWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation), false); +// } + +// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "3"); +} + +void InotifyWatch::run() +{ + m_inotifyFd = inotify_init(); + if (m_inotifyFd > 0) { + qDebug()<<"Inotify init success!"; + } else { + Q_ASSERT_X(0, "InotifyWatch", "Failed to initialize inotify"); + } + + this->addWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation)); + this->setPath(QStandardPaths::writableLocation(QStandardPaths::HomeLocation)); + this->firstTraverse(); + + int fifo_fd; + char buffer[2]; + memset(buffer, 0, sizeof(buffer)); + fifo_fd = open(UKUI_SEARCH_PIPE_PATH, O_RDWR); + if(fifo_fd == -1) { + qWarning() << "Open fifo error\n"; + assert(false); + } + int retval = read(fifo_fd, buffer, sizeof(buffer)); + if(retval == -1) { + qWarning() << "read error\n"; + assert(false); + } + qDebug("Read fifo[%s]", buffer); + + qDebug("Read data ok"); + close(fifo_fd); + if(buffer[0] & 0x1) { + qDebug("Data confirmed\n"); + } + unlink(UKUI_SEARCH_PIPE_PATH); + + while(FileUtils::SearchMethod::INDEXSEARCH == FileUtils::searchMethod) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(m_inotifyFd, &fds); + int rc; + rc = select(m_inotifyFd + 1, &fds, NULL, NULL, NULL); + if(rc > 0) { + int avail; + if (ioctl(m_inotifyFd, FIONREAD, &avail) == EINVAL) { + qWarning() << "Did not receive an entire inotify event."; + return; + } + + char* buf = (char*)malloc(avail); + memset(buf, 0x00, avail); + + const ssize_t len = read(m_inotifyFd, buf, avail); + if(len != avail) { + qWarning()<<"read event error"; + // IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1"); + } + + int i = 0; + while (i < len) { + const struct inotify_event* event = (struct inotify_event*)&buf[i]; + if(event->name[0] != '.') { + // qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask; + // qDebug("mask:0x%x,",event->mask); + break; + } + i += sizeof(struct inotify_event) + event->len; + } + if(i < len ) { + qDebug() << "fork"; + slotEvent(buf, len); + free(buf); + } + } + } + if(FileUtils::SearchMethod::DIRECTSEARCH == FileUtils::searchMethod) { + IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "3"); + removeWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation), false); + } +// fcntl(m_inotifyFd, F_SETFD, FD_CLOEXEC); +// m_notifier = new QSocketNotifier(m_inotifyFd, QSocketNotifier::Read); +// connect(m_notifier, &QSocketNotifier::activated, this, &InotifyWatch::slotEvent, Qt::DirectConnection); +// exec(); +} + +void InotifyWatch::slotEvent(char *buf, ssize_t len) +{ +// eventProcess(socket); + ++FileUtils::_index_status; + if(FileUtils::SearchMethod::INDEXSEARCH == FileUtils::searchMethod) { + pid_t pid; + pid = fork(); + if(pid == 0) { + prctl(PR_SET_PDEATHSIG, SIGTERM); + prctl(PR_SET_NAME, "inotify-index"); + + this->eventProcess(buf, len); + fd_set read_fds; + int rc; + timeval* read_timeout = (timeval*)malloc(sizeof(timeval)); + read_timeout->tv_sec = 40; + read_timeout->tv_usec = 0; + for(;;) { + FD_ZERO(&read_fds); + FD_SET(m_inotifyFd, &read_fds); + qDebug() << read_timeout->tv_sec; + rc = select(m_inotifyFd + 1, &read_fds, NULL, NULL, read_timeout); + if(rc < 0) { + // error + qWarning() << "select result < 0, error!"; + IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1"); + assert(false); + } else if(rc == 0) { + qDebug() << "select timeout!"; + ::free(read_timeout); + + QBuffer buffer; + QDataStream out(&buffer); + if (m_sharedMemory->isAttached()) { + m_sharedMemory->detach(); + } + buffer.open(QBuffer::ReadWrite); + out << currentPath; + int size = buffer.size(); + if (!m_sharedMemory->create(size)) { + qDebug() << "Create sharedMemory Error: " << m_sharedMemory->errorString(); + } else { + m_sharedMemory->lock(); + char *to = static_cast(m_sharedMemory->data()); + const char *from = buffer.data().constData(); + memcpy(to, from, qMin(size, m_sharedMemory->size())); + m_sharedMemory->unlock(); + } + // GlobalSettings::getInstance()->forceSync(); + PendingFileQueue::getInstance()->forceFinish(); + PendingFileQueue::getInstance()->~PendingFileQueue(); + ::_exit(0); + } else { + qDebug() << "Select remain:" <tv_sec; + this->eventProcess(m_inotifyFd); + qDebug() << "Select remain:" <tv_sec; + } + } + } else if(pid > 0) { + waitpid(pid, NULL, 0); + if (!m_sharedMemory->attach()) { + qDebug() << "SharedMemory attach Error: " << m_sharedMemory->errorString(); + } else { + QBuffer buffer; + QDataStream in(&buffer); + QMap pathMap; + m_sharedMemory->lock(); + buffer.setData(static_cast(m_sharedMemory->constData()), m_sharedMemory->size()); + buffer.open(QBuffer::ReadWrite); + in >> pathMap; + m_sharedMemory->unlock(); + m_sharedMemory->detach(); + currentPath = pathMap; + } + --FileUtils::_index_status; + } else { + assert(false); + } + } +} + +char * InotifyWatch::filter() +{ + int avail; + if (ioctl(m_inotifyFd, FIONREAD, &avail) == EINVAL) { + qWarning() << "Did not receive an entire inotify event."; + return NULL; + } + + char* buffer = (char*)malloc(avail); + memset(buffer, 0x00, avail); + + const int len = read(m_inotifyFd, buffer, avail); + if(len != avail) { + qWarning()<<"read event error"; +// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1"); + } + + int i = 0; + while (i < len) { + const struct inotify_event* event = (struct inotify_event*)&buffer[i]; + if(event->name[0] == '.') { + // qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask; + // qDebug("mask:0x%x,",event->mask); + i += sizeof(struct inotify_event) + event->len; + return buffer; + } + } + free(buffer); + return NULL; +} +void InotifyWatch::eventProcess(int socket) +{ + qDebug()<< "Enter eventProcess!"; + int avail; + if (ioctl(socket, FIONREAD, &avail) == EINVAL) { + qWarning() << "Did not receive an entire inotify event."; + return; + } + + char* buffer = (char*)malloc(avail); + memset(buffer, 0x00, avail); + + const ssize_t len = read(socket, buffer, avail); + if(len != avail) { + qWarning()<<"read event error"; +// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1"); + } + int i = 0; + while (i < len) { + const struct inotify_event* event = (struct inotify_event*)&buffer[i]; + if(event->name[0] != '.') { + qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask; + qDebug("mask:0x%x,",event->mask); + break; + } + i += sizeof(struct inotify_event) + event->len; + } + if(i >= len) { + qDebug() << "There is nothing to do!"; + return; + } + eventProcess(buffer, len); + free(buffer); +} + +void InotifyWatch::eventProcess(const char *buffer, ssize_t len) +{ + qDebug()<< "Begin eventProcess! len:" << len; + IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "0"); + + char * p = const_cast(buffer); + while (p < buffer + len) { + const struct inotify_event* event = reinterpret_cast(p); +// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask; +// qDebug("mask:0x%x,",event->mask); + if(event->name[0] != '.') { + QString path = currentPath[event->wd] + '/' + event->name; + //Create top dir first, traverse it last. + if(event->mask & IN_CREATE) { + qDebug() << "IN_CREATE"; + PendingFile f(path); + if(event->mask & IN_ISDIR) { + f.setIsDir(); + } + PendingFileQueue::getInstance(this)->enqueue(f); + + if(event->mask & IN_ISDIR) { + if(!QFileInfo(path).isSymLink()){ + addWatch(path); + setPath(path); + Traverse(); + } + } + goto next; + + } + + if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)) { + qDebug() << "IN_DELETE or IN_MOVED_FROM"; + if(event->mask & IN_ISDIR) { + removeWatch(path); + } else { + PendingFile f(path); + f.setDeleted(); + PendingFileQueue::getInstance()->enqueue(f); + } + p += sizeof(struct inotify_event) + event->len; + continue; + } + if(event->mask & IN_MODIFY) { + qDebug() << "IN_MODIFY"; + if(!(event->mask & IN_ISDIR)) { + PendingFileQueue::getInstance()->enqueue(PendingFile(path)); + } + goto next; + + } + + if(event->mask & IN_MOVED_TO) { + qDebug() << "IN_MOVED_TO"; + if(event->mask & IN_ISDIR) { + removeWatch(path); + + PendingFile f(path); + f.setIsDir(); + PendingFileQueue::getInstance()->enqueue(f); + + if(!QFileInfo(path).isSymLink()){ + addWatch(path); + setPath(path); + Traverse(); + } + } else { + //Enqueue a deleted file to merge. + PendingFile f(path); + f.setDeleted(); + PendingFileQueue::getInstance()->enqueue(f); + //Enqueue a new one. + PendingFileQueue::getInstance()->enqueue(PendingFile(path)); + } + goto next; + } + } +next: + p += sizeof(struct inotify_event) + event->len; + } + qDebug()<< "Finish eventProcess!"; +} + + diff --git a/libsearch/index/inotify-watch.h b/libsearch/index/inotify-watch.h new file mode 100644 index 0000000..eb21587 --- /dev/null +++ b/libsearch/index/inotify-watch.h @@ -0,0 +1,52 @@ +#ifndef INOTIFYWATCH_H +#define INOTIFYWATCH_H + +#include +#include +#include +#include +#include +#include + +#include "traverse_bfs.h" +#include "ukui-search-qdbus.h" +#include "index-status-recorder.h" +#include "file-utils.h" +#include "first-index.h" +#include "pending-file-queue.h" +#include "common.h" +namespace Zeeker { +class InotifyWatch : public QThread, public Traverse_BFS +{ + Q_OBJECT +public: + static InotifyWatch* getInstance(const QString& path); + + bool addWatch(const QString &path); + bool removeWatch(const QString &path, bool removeFromDatabase = true); + virtual void DoSomething(const QFileInfo &info) final; + + void firstTraverse(); + void stopWatch(); +protected: + void run() override; + +private Q_SLOTS: + void slotEvent(char *buf, ssize_t len); +private: + explicit InotifyWatch(const QString& path); + ~InotifyWatch(); + char * filter(); + void eventProcess(int socket); + void eventProcess(const char *buffer, ssize_t len); + + int m_inotifyFd; + QSocketNotifier* m_notifier = nullptr; + QSharedMemory *m_sharedMemory = nullptr; + QMap currentPath; + QMutex m_mutex; + + +}; +} +#endif // INOTIFYWATCH_H diff --git a/libsearch/index/pending-file-queue.cpp b/libsearch/index/pending-file-queue.cpp new file mode 100644 index 0000000..c1dbe32 --- /dev/null +++ b/libsearch/index/pending-file-queue.cpp @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2021, KylinSoft Co., Ltd. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: zhangpengfei + * + */ +#include "pending-file-queue.h" +#include +using namespace Zeeker; +static PendingFileQueue *global_instance_pending_file_queue = nullptr; +PendingFileQueue::PendingFileQueue(QObject *parent) : QThread(parent) +{ + this->start(); + + m_cacheTimer = new QTimer; + m_minProcessTimer = new QTimer; + + m_cacheTimer->setInterval(10*1000); + m_cacheTimer->setSingleShot(true); + m_minProcessTimer->setInterval(500); + m_minProcessTimer->setSingleShot(true); + + m_cacheTimer->moveToThread(this); + m_minProcessTimer->moveToThread(this); + +// connect(this, &PendingFileQueue::cacheTimerStart, m_cacheTimer, f, Qt::DirectConnection); +// connect(this, &PendingFileQueue::minProcessTimerStart, m_minProcessTimer, f,Qt::DirectConnection); + connect(this, SIGNAL(cacheTimerStart()), m_cacheTimer, SLOT(start())); + connect(this, SIGNAL(minProcessTimerStart()), m_minProcessTimer, SLOT(start())); + + connect(m_cacheTimer, &QTimer::timeout, this, &PendingFileQueue::processCache, Qt::DirectConnection); + connect(m_minProcessTimer, &QTimer::timeout, this, &PendingFileQueue::processCache, Qt::DirectConnection); + +} + +PendingFileQueue *PendingFileQueue::getInstance(QObject *parent) +{ + if (!global_instance_pending_file_queue) { + global_instance_pending_file_queue = new PendingFileQueue(parent); + } + return global_instance_pending_file_queue; +} + +PendingFileQueue::~PendingFileQueue() +{ + if(m_cacheTimer) { + delete m_cacheTimer; + m_cacheTimer = nullptr; + } + if(m_minProcessTimer) { + delete m_minProcessTimer; + m_minProcessTimer = nullptr; + } + + IndexGenerator::getInstance()->~IndexGenerator(); +} + +void PendingFileQueue::forceFinish() +{ + QThread::msleep(600); + this->quit(); + this->wait(); +} +void PendingFileQueue::enqueue(const PendingFile &file) +{ + qDebug() << "enqueuq file: " << file.path(); + m_mutex.lock(); + // Remove all indexs of files under a dir which is to about be deleted,but keep delete signals. + // Because our datebase need to delete those indexs one by one. + if(file.shouldRemoveIndex() && file.isDir()) { + const auto keepFile = [&file](const PendingFile& pending) { + return (!pending.path().startsWith(file.path()) || pending.shouldRemoveIndex()); + }; + const auto end = m_cache.end(); + const auto droppedFilesBegin = std::stable_partition(m_cache.begin(), end, keepFile); + m_cache.erase(droppedFilesBegin, end); + } + + if(file.shouldRemoveIndex()) { + m_cache.removeOne(file); + } + int i = m_cache.indexOf(file); + if (i == -1) { + qDebug() << "insert file" << file.path() << file.shouldRemoveIndex(); + m_cache << file; + } else { + qDebug() << "merge file" << file.path() << file.shouldRemoveIndex(); + m_cache[i].merge(file); + } + + if(!m_cacheTimer->isActive()) { + qDebug()<<"m_cacheTimer-----start!!"; +// m_cacheTimer->start(); + Q_EMIT cacheTimerStart(); + } + Q_EMIT minProcessTimerStart(); +// m_minProcessTimer->start(); + qDebug()<<"m_minProcessTimer-----start!!"; + m_mutex.unlock(); + qDebug() << "Current cache-------------"; + for(PendingFile i : m_cache) { + qDebug() << "|" << i.path(); + qDebug() << "|" <updateIndex(&m_pendingFiles); + m_mutex.lock(); + if(m_cache.isEmpty()) { + IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2"); + } + m_mutex.unlock(); + m_pendingFiles.clear(); + m_pendingFiles.squeeze(); + malloc_trim(0); + qDebug()<< "Finish processCache!"; + return; +} diff --git a/libsearch/index/pending-file-queue.h b/libsearch/index/pending-file-queue.h new file mode 100644 index 0000000..dc75b0e --- /dev/null +++ b/libsearch/index/pending-file-queue.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2021, KylinSoft Co., Ltd. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: zhangpengfei + * + */ +#ifndef PENDINGFILEQUEUE_H +#define PENDINGFILEQUEUE_H + +#include +#include +#include +#include +#include +#include "pending-file.h" +#include "index-generator.h" + +namespace Zeeker { +class PendingFileQueue : public QThread +{ + Q_OBJECT +public: + static PendingFileQueue *getInstance(QObject *parent = nullptr); + + ~PendingFileQueue(); + //This method will block until current cache has been processed. + //Do not do enqueue operation in other thread while this method is running. + void forceFinish(); + void enqueue(const PendingFile& file); + QTimer *m_cacheTimer = nullptr; + QTimer *m_minProcessTimer = nullptr; + +protected: + void run() override; +Q_SIGNALS: + void cacheTimerStart(); + void minProcessTimerStart(); +private: + void processCache(); + explicit PendingFileQueue(QObject *parent = nullptr); + + QVector m_cache; + QVector m_pendingFiles; + QMutex m_mutex; + QMutex m_timeoutMutex; + + QThread *m_timerThread = nullptr; + bool m_timeout = false; + +}; +} +#endif // PENDINGFILEQUEUE_H diff --git a/libsearch/index/pending-file.cpp b/libsearch/index/pending-file.cpp new file mode 100644 index 0000000..aa30720 --- /dev/null +++ b/libsearch/index/pending-file.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2021, KylinSoft Co., Ltd. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: zhangpengfei + * + */ +#include "pending-file.h" +using namespace Zeeker; +PendingFile::PendingFile(const QString &path) + : m_path(path) + , m_deleted(false) + , m_modified(false) + , m_isDir(false) +{ + +} + +QString PendingFile::path() const +{ + return m_path; +} + +void PendingFile::setPath(const QString& path) +{ + if (path.endsWith(QLatin1Char('/'))) { + m_path = path.mid(0, m_path.length() - 1); + return; + } + m_path = path; +} + +//bool PendingFile::isNewFile() const +//{ +// return m_created; +//} + +//bool PendingFile::shouldIndexContents() const +//{ +// if (m_created || m_modified) { +// return true; +// } +// return false; +//} + +bool PendingFile::isDir() const +{ + return m_isDir; +} + +bool PendingFile::shouldRemoveIndex() const +{ + return m_deleted; +} + +void PendingFile::merge(const PendingFile& file) +{ +// m_created |= file.m_created; + m_modified = file.m_modified; + m_deleted = file.m_deleted; +} + +void PendingFile::printFlags() const +{ +// qDebug() << "Created:" << m_created; + qDebug() << "Deleted:" << m_deleted; + qDebug() << "Modified:" << m_modified; + qDebug() << "Is dir:" << m_isDir; +} diff --git a/libsearch/index/pending-file.h b/libsearch/index/pending-file.h new file mode 100644 index 0000000..41555c6 --- /dev/null +++ b/libsearch/index/pending-file.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2021, KylinSoft Co., Ltd. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: zhangpengfei + * + */ +#ifndef PENDINGFILE_H +#define PENDINGFILE_H + +#include +#include +namespace Zeeker { +/** + * Represents a file/folder which needs to be indexed. + */ +class PendingFile +{ +public: + explicit PendingFile(const QString& path = QString()); + + QString path() const; + void setPath(const QString& path); + void setIsDir(){ m_isDir = true; } + void setModified() { m_modified = true; } +// void setCreated() { m_created = true; } + void setDeleted() { m_deleted = true; } + bool shouldRemoveIndex() const; +// bool shouldIndexContents() const; + bool isDir() const; + + bool operator == (const PendingFile& rhs) const { + return (m_path == rhs.m_path); + } + + /** + * Takes a PendingFile \p file and merges its flags into + * the current PendingFile + */ + void merge(const PendingFile& file); + +private: + QString m_path; + +// bool m_created : 1; + bool m_deleted : 1; + bool m_modified : 1; + bool m_isDir : 1; + + void printFlags() const; +}; +} +#endif // PENDINGFILE_H diff --git a/libsearch/index/searchmethodmanager.cpp b/libsearch/index/searchmethodmanager.cpp index 0726abd..dc073cb 100644 --- a/libsearch/index/searchmethodmanager.cpp +++ b/libsearch/index/searchmethodmanager.cpp @@ -1,25 +1,34 @@ #include "searchmethodmanager.h" using namespace Zeeker; +SearchMethodManager::SearchMethodManager() +{ + m_iw = InotifyWatch::getInstance(HOME_PATH); +} + void SearchMethodManager::searchMethod(FileUtils::SearchMethod sm) { qWarning() << "searchMethod start: " << static_cast(sm); if(FileUtils::SearchMethod::INDEXSEARCH == sm || FileUtils::SearchMethod::DIRECTSEARCH == sm) { FileUtils::searchMethod = sm; } else { - printf("enum class error!!!\n"); qWarning("enum class error!!!\n"); } if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::_index_status) { qWarning() << "start first index"; -// m_fi = FirstIndex("/home/zhangzihao/Desktop"); m_fi.start(); qWarning() << "start inotify index"; // InotifyIndex ii("/home"); // ii.start(); - this->m_ii = InotifyIndex::getInstance("/home"); - if(!this->m_ii->isRunning()) { - this->m_ii->start(); +// this->m_ii = InotifyIndex::getInstance("/home"); +// if(!this->m_ii->isRunning()) { +// this->m_ii->start(); +// } + if(!this->m_iw->isRunning()) { + this->m_iw->start(); } qDebug() << "Search method has been set to INDEXSEARCH"; } + if(FileUtils::SearchMethod::DIRECTSEARCH == sm) { + m_iw->stopWatch(); + } qWarning() << "searchMethod end: " << static_cast(FileUtils::searchMethod); } diff --git a/libsearch/index/searchmethodmanager.h b/libsearch/index/searchmethodmanager.h index 1c842ca..f95421b 100644 --- a/libsearch/index/searchmethodmanager.h +++ b/libsearch/index/searchmethodmanager.h @@ -2,15 +2,17 @@ #define SEARCHMETHODMANAGER_H #include "first-index.h" -#include "inotify-index.h" +//#include "inotify-index.h" +#include "inotify-watch.h" namespace Zeeker { class SearchMethodManager { public: - SearchMethodManager() = default; + SearchMethodManager(); void searchMethod(FileUtils::SearchMethod sm); private: FirstIndex m_fi; - InotifyIndex* m_ii; +// InotifyIndex* m_ii; + InotifyWatch *m_iw = nullptr; }; }