From e2fd50baa14119eed253df489d1a6e077b96bde5 Mon Sep 17 00:00:00 2001 From: zhangzihao Date: Sun, 10 Jan 2021 09:01:22 +0800 Subject: [PATCH] add inotify create or delete index and fix first index makes the size of database increase unlimitedly --- libsearch/global-settings.cpp | 7 +- libsearch/index/first-index.cpp | 15 ++- libsearch/index/index-generator.cpp | 1 + libsearch/index/index-generator.h | 2 +- libsearch/index/index.pri | 2 + libsearch/index/inotify-index.cpp | 141 ++++++++++++++++++++++++++++ libsearch/index/inotify-index.h | 40 ++++++++ libsearch/index/inotify.cpp | 15 +-- libsearch/index/inotify.h | 2 +- libsearch/libsearch.h | 1 + src/main.cpp | 6 +- 11 files changed, 218 insertions(+), 14 deletions(-) create mode 100644 libsearch/index/inotify-index.cpp create mode 100644 libsearch/index/inotify-index.h diff --git a/libsearch/global-settings.cpp b/libsearch/global-settings.cpp index f8037d3..a0db96b 100644 --- a/libsearch/global-settings.cpp +++ b/libsearch/global-settings.cpp @@ -103,15 +103,18 @@ QStringList GlobalSettings::getBlockDirs() return m_block_dirs_settings->allKeys(); } +//here should be override +//MouseZhangZh void GlobalSettings::setValue(const QString &key, const QVariant &value) { m_cache.insert(key, value); QtConcurrent::run([=]() { - if (m_mutex.tryLock(1000)) { +// if (m_mutex.tryLock(1000)) { + m_mutex.lock(); m_settings->setValue(key, value); m_settings->sync(); m_mutex.unlock(); - } +// } }); } diff --git a/libsearch/index/first-index.cpp b/libsearch/index/first-index.cpp index d377ffd..2f0384f 100644 --- a/libsearch/index/first-index.cpp +++ b/libsearch/index/first-index.cpp @@ -14,9 +14,15 @@ FirstIndex::FirstIndex(const QString& path) : Traverse_BFS(path) if (indexDataBaseStatus == "" || contentIndexDataBaseStatus == ""){ this->bool_dataBaseExist = false; } + else{ + this->bool_dataBaseExist = true; + } if (indexDataBaseStatus != "2" || contentIndexDataBaseStatus != "2"){ this->bool_dataBaseStatusOK = false; } + else{ + this->bool_dataBaseStatusOK = true; + } this->q_index = new QQueue>(); this->q_content_index = new QQueue(); @@ -26,10 +32,13 @@ FirstIndex::FirstIndex(const QString& path) : Traverse_BFS(path) FirstIndex::~FirstIndex() { + qDebug() << "~FirstIndex"; delete this->q_index; this->q_index = nullptr; - this->q_content_index = nullptr; delete this->q_content_index; + this->q_content_index = nullptr; + delete this->p_indexGenerator; + this->p_indexGenerator; // delete this->mlm; // this->mlm = nullptr; } @@ -95,6 +104,10 @@ void FirstIndex::run(){ mutex2.unlock(); mutex3.unlock(); qDebug() << "first index end;"; + //don't use it now!!!! + //MouseZhangZh +// this->~FirstIndex(); +// qDebug() << "~FirstIndex end;"; this->quit(); // this->wait(); } diff --git a/libsearch/index/index-generator.cpp b/libsearch/index/index-generator.cpp index 7d9dd84..982703e 100644 --- a/libsearch/index/index-generator.cpp +++ b/libsearch/index/index-generator.cpp @@ -123,6 +123,7 @@ IndexGenerator::~IndexGenerator() if(m_database_content) delete m_database_content; GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE,"2"); + GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE,"2"); GlobalSettings::getInstance()->setValue(INDEX_GENERATOR_NORMAL_EXIT,"2"); } diff --git a/libsearch/index/index-generator.h b/libsearch/index/index-generator.h index ebc53ed..20d1e58 100644 --- a/libsearch/index/index-generator.h +++ b/libsearch/index/index-generator.h @@ -15,6 +15,7 @@ class IndexGenerator : public QObject Q_OBJECT public: static IndexGenerator *getInstance(bool rebuild = false); + ~IndexGenerator(); bool setIndexdataPath(); bool isIndexdataExist(); //for search test @@ -29,7 +30,6 @@ public Q_SLOTS: private: explicit IndexGenerator(bool rebuild = false,QObject *parent = nullptr); - ~IndexGenerator(); //For file name index void HandlePathList(QList> *messageList); //For file content index diff --git a/libsearch/index/index.pri b/libsearch/index/index.pri index 63343ce..14d592e 100644 --- a/libsearch/index/index.pri +++ b/libsearch/index/index.pri @@ -9,6 +9,7 @@ HEADERS += \ $$PWD/first-index.h \ $$PWD/index-generator.h \ # $$PWD/inotify-manager.h \ + $$PWD/inotify-index.h \ $$PWD/inotify.h \ $$PWD/messagelist-manager.h \ $$PWD/traverse_bfs.h \ @@ -25,6 +26,7 @@ SOURCES += \ $$PWD/first-index.cpp \ $$PWD/index-generator.cpp \ # $$PWD/inotify-manager.cpp \ + $$PWD/inotify-index.cpp \ $$PWD/inotify.cpp \ $$PWD/messagelist-manager.cpp \ $$PWD/test-Inotify-Manager.cpp \ diff --git a/libsearch/index/inotify-index.cpp b/libsearch/index/inotify-index.cpp new file mode 100644 index 0000000..269a7af --- /dev/null +++ b/libsearch/index/inotify-index.cpp @@ -0,0 +1,141 @@ +#include "inotify-index.h" + + +InotifyIndex::InotifyIndex(const QString& path) : Traverse_BFS(path) +{ + /*-------------ukuisearchdbus Test start-----------------*/ + qDebug() << "setInotifyMaxUserWatches start"; + UkuiSearchQDBus usQDBus; + usQDBus.setInotifyMaxUserWatches(); + qDebug() << "setInotifyMaxUserWatches end"; + + /*-------------ukuisearchdbus Test End-----------------*/ + m_fd = inotify_init(); + qDebug() << "m_fd----------->" <AddWatch("/home"); + this->Traverse(); +} + +InotifyIndex::~InotifyIndex() +{ + +} + +void InotifyIndex::DoSomething(const QFileInfo& fileInfo){ + if(fileInfo.isDir()){ + this->AddWatch(fileInfo.absoluteFilePath()); + } +} + +bool InotifyIndex::AddWatch(const QString &path){ + int ret = inotify_add_watch(m_fd, path.toStdString().c_str(), (IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | IN_DELETE)); + if (ret == -1) { + qDebug() << "AddWatch error:" << path; + return false; + } + Q_ASSERT(ret != -1); + currentPath[ret] = path; + //qDebug() << "Watch:" << path; + return true; +} + +bool InotifyIndex::RemoveWatch(const QString &path){ + int ret = inotify_rm_watch(m_fd, currentPath.key(path)); + if (ret){ + qDebug() << "remove path error"; + return false; + } + Q_ASSERT(ret != 0); + + for (QMap::Iterator i = currentPath.begin(); i != currentPath.end();){ + if (i.value().length() > path.length()){ + if (i.value().mid(0, path.length()) == path){ +// qDebug() << i.value(); + /*--------------------------------*/ + //在此调用删除索引 + IndexGenerator::getInstance()->deleteAllIndex(new QStringList(path)); + /*--------------------------------*/ + currentPath.erase(i++); + } + else{ + i++; + } + } + else{ + i++; + } + } +// qDebug() << path; + //这个貌似不用删，先mark一下 + //currentPath.remove(currentPath.key(path)); + return true; +} + + +void InotifyIndex::run(){ + + char * p; + char buf[BUF_LEN] __attribute__((aligned(8))); + + ssize_t numRead; + + QList>* indexList = new QList>(); + QList* contentIndexList = new QList(); + + for (;;) { /* Read events forever */ + numRead = read(m_fd, buf, BUF_LEN); + if (numRead == 0) { + qDebug() << "read() from inotify fd returned 0!"; + } + if (numRead == -1) { + qDebug() << "read"; + } + qDebug() << "Read " << numRead << " bytes from inotify fd"; + + /* Process all of the events in buffer returned by read() */ + + for (p = buf; p < buf + numRead;) { + struct inotify_event * event = reinterpret_cast(p); + if(event->name[0] != '.'){ + + //传创建或移动过来的文件路径 + if((event->mask & IN_CREATE) | (event->mask & IN_MOVED_TO)){ + //添加监视要先序遍历，先添加top节点 + if (event->mask & IN_ISDIR){ + AddWatch(currentPath[event->wd] + '/' + event->name); + this->setPath(currentPath[event->wd] + '/' + event->name); + Traverse(); + } + + /*--------------------------------*/ +// IndexGenerator::getInstance()->creatAllIndex(QQueue>(QVector() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0"))); + indexList->append(QVector() << QString(event->name) << QString(currentPath[event->wd] + '/' + event->name) << QString((event->mask & IN_ISDIR) ? "1" : "0")); + IndexGenerator::getInstance()->creatAllIndex(indexList); + indexList->clear(); + for (auto i : this->targetFileTypeVec){ + if (QString(currentPath[event->wd] + '/' + event->name).endsWith(i)){ + contentIndexList->append(QString(currentPath[event->wd] + '/' + event->name)); + IndexGenerator::getInstance()->creatAllIndex(contentIndexList); + contentIndexList->clear(); + } + } + /*--------------------------------*/ + } + else if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){ + if (event->mask & IN_ISDIR){ + RemoveWatch(currentPath[event->wd] + '/' + event->name); + } + IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); + } + /*--------------------------------*/ + } + p += sizeof(struct inotify_event) + event->len; + } + } + + delete indexList; + indexList = nullptr; + delete contentIndexList; + contentIndexList = nullptr; +} diff --git a/libsearch/index/inotify-index.h b/libsearch/index/inotify-index.h new file mode 100644 index 0000000..c9ae1ec --- /dev/null +++ b/libsearch/index/inotify-index.h @@ -0,0 +1,40 @@ +#ifndef INOTIFYINDEX_H +#define INOTIFYINDEX_H + +#include +#include +#include +#include "index-generator.h" +#include "traverse_bfs.h" +#include "ukui-search-qdbus.h" + +#define BUF_LEN 1024 + +class InotifyIndex : public QThread, public Traverse_BFS +{ + Q_OBJECT +public: + InotifyIndex(const QString&); + ~InotifyIndex(); + + bool AddWatch(const QString&); + bool RemoveWatch(const QString&); + virtual void DoSomething(const QFileInfo &) final; +protected: + void run() override; +private: + QString* m_watch_path; + int m_fd; + + QMap currentPath; + const QVector targetFileTypeVec ={ +// QString(".doc"), + QString(".docx"), +// QString(".ppt"), +// QString(".pptx"), +// QString(".xls"), +// QString(".xlsx"), + QString(".txt")}; +}; + +#endif // INOTIFYINDEX_H diff --git a/libsearch/index/inotify.cpp b/libsearch/index/inotify.cpp index 0c093e8..81293b3 100644 --- a/libsearch/index/inotify.cpp +++ b/libsearch/index/inotify.cpp @@ -33,7 +33,7 @@ InotifyManagerRefact::InotifyManagerRefact(const QString& path) : Traverse_BFS(p num2string.insert(IN_UNMOUNT, "IN_UNMOUNT"); num2string.insert(IN_Q_OVERFLOW, "IN_Q_OVERFLOW"); num2string.insert(IN_IGNORED, "IN_IGNORED"); - this->mlm = new MessageListManager(); +// this->mlm = new MessageListManager(); this->AddWatch("/home"); this->Traverse(); @@ -44,14 +44,14 @@ InotifyManagerRefact::InotifyManagerRefact(const QString& path) : Traverse_BFS(p } InotifyManagerRefact::~InotifyManagerRefact(){ - delete this->mlm; - this->mlm = nullptr; +// delete this->mlm; +// this->mlm = nullptr; // delete dirPath; // dirPath = nullptr; } void InotifyManagerRefact::DoSomething(const QFileInfo& fileInfo){ - this->mlm->AddMessage(QVector() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir()?"1":"0")); +// this->mlm->AddMessage(QVector() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir()?"1":"0")); // if(QString(bool((fileInfo.isDir()))) == QString("1")) // qDebug()<<"bool((fileInfo.isDir())"<mlm->AddMessage(QVector() << "PLog" << "/home/zpf/baidunetdisk/PLog" << "1"); @@ -65,7 +65,7 @@ void InotifyManagerRefact::DoSomething(const QFileInfo& fileInfo){ void InotifyManagerRefact::SendRestMessage() { - this->mlm->SendMessage(); +// this->mlm->SendMessage(); } bool InotifyManagerRefact::AddWatch(const QString &path){ @@ -73,6 +73,7 @@ bool InotifyManagerRefact::AddWatch(const QString &path){ // qDebug() << "m_fd: " <creatAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); - this->mlm->AddMessage(QVector() << event->name << (currentPath[event->wd] + '/' + event->name) << QString(bool((event->mask & IN_ISDIR)))); - this->mlm->SendMessage(); +// this->mlm->AddMessage(QVector() << event->name << (currentPath[event->wd] + '/' + event->name) << QString(bool((event->mask & IN_ISDIR)))); +// this->mlm->SendMessage(); // } } else if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){ diff --git a/libsearch/index/inotify.h b/libsearch/index/inotify.h index 6e5818b..dff0d7e 100644 --- a/libsearch/index/inotify.h +++ b/libsearch/index/inotify.h @@ -29,7 +29,7 @@ private: int m_fd; QMap currentPath; QMap num2string; - MessageListManager* mlm; +// MessageListManager* mlm; QMap* dirPath; }; diff --git a/libsearch/libsearch.h b/libsearch/libsearch.h index 2205333..5819914 100644 --- a/libsearch/libsearch.h +++ b/libsearch/libsearch.h @@ -13,6 +13,7 @@ #include "index/filetypefilter.h" #include "index/ukui-search-qdbus.h" +#include "index/inotify-index.h" class LIBSEARCH_EXPORT GlobalSearch { diff --git a/src/main.cpp b/src/main.cpp index cde613d..7d5f487 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -94,8 +94,10 @@ int main(int argc, char *argv[]) qInstallMessageHandler(messageOutput); qDebug() << "main start"; - FirstIndex* fi = new FirstIndex("/home"); - fi->start(); + FirstIndex fi("/home"); + fi.start(); + InotifyIndex ii("/home"); + ii.start(); /*-------------ukuisearchdbus Test start-----------------*/ // UkuiSearchQDBus usQDBus; // usQDBus.setInotifyMaxUserWatches();