From ea7e44cf9afef8304850df8d4292b455ca308061 Mon Sep 17 00:00:00 2001 From: zhangpengfei Date: Mon, 21 Dec 2020 18:50:54 +0800 Subject: [PATCH] Add files about index creation. --- control/control.pri | 1 + file-utils.cpp | 10 ++ file-utils.h | 14 +++ index/index-generator.cpp | 192 ++++++++++++++++++++++++++++++ index/index-generator.h | 42 +++++++ index/index.pri | 13 +++ index/inotify-manager.cpp | 219 +++++++++++++++++++++++++++++++++++ index/inotify-manager.h | 40 +++++++ index/messagelistmanager.cpp | 35 ++++++ index/messagelistmanager.h | 26 +++++ index/testInotifyManager.cpp | 38 ++++++ model/model.pri | 1 + src/src.pri | 10 ++ ukui-search.pro | 19 +-- 14 files changed, 653 insertions(+), 7 deletions(-) create mode 100644 control/control.pri create mode 100644 file-utils.cpp create mode 100644 file-utils.h create mode 100644 index/index-generator.cpp create mode 100644 index/index-generator.h create mode 100644 index/index.pri create mode 100644 index/inotify-manager.cpp create mode 100644 index/inotify-manager.h create mode 100644 index/messagelistmanager.cpp create mode 100644 index/messagelistmanager.h create mode 100644 index/testInotifyManager.cpp create mode 100644 model/model.pri create mode 100644 src/src.pri diff --git a/control/control.pri b/control/control.pri new file mode 100644 index 0000000..aba082d --- /dev/null +++ b/control/control.pri @@ -0,0 +1 @@ +INCLUDEPATH += $$PWD diff --git a/file-utils.cpp b/file-utils.cpp new file mode 100644 index 0000000..a0efa16 --- /dev/null +++ b/file-utils.cpp @@ -0,0 +1,10 @@ +#include "file-utils.h" + +FileUtils::FileUtils() +{ +} + +std::string FileUtils::makeDocUterm(QString *path) +{ + return QCryptographicHash::hash(path->toUtf8(),QCryptographicHash::Md5).toStdString(); +} diff --git a/file-utils.h b/file-utils.h new file mode 100644 index 0000000..b608ae9 --- /dev/null +++ b/file-utils.h @@ -0,0 +1,14 @@ +#ifndef FILEUTILS_H +#define FILEUTILS_H +#include +#include + +class FileUtils +{ +public: + static std::string makeDocUterm(QString *path); +private: + FileUtils(); +}; + +#endif // FILEUTILS_H diff --git a/index/index-generator.cpp b/index/index-generator.cpp new file mode 100644 index 0000000..c2a88dc --- /dev/null +++ b/index/index-generator.cpp @@ -0,0 +1,192 @@ +#include "index-generator.h" +#include +#include +#include + +using namespace std; + +#define INDEX_PATH "/home/zpf/.config/org.ukui/index_data" +static IndexGenerator *global_instance = nullptr; + +IndexGenerator *IndexGenerator::getInstance() +{ + if (!global_instance) { + global_instance = new IndexGenerator; + } + return global_instance; +} + +bool IndexGenerator::creatAllIndex(QStringList *pathlist) +{ + HandlePathList(pathlist); + try + { + m_indexer = new Xapian::TermGenerator(); + m_indexer->set_database(*m_datebase); + m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING); + m_indexer->set_stemming_strategy(Xapian::TermGenerator::STEM_SOME); + + QMap::const_iterator i; + QString *indexStr; + QString *docStr; + for(i=m_index_map->constBegin();i!=m_index_map->constEnd();++i) + { + docStr = new QString(i.key()); + indexStr = new QString(i.value()); + CreatIndex(indexStr,docStr); + } + m_datebase->commit(); + } + catch(const Xapian::Error &e) + { + qDebug()<<"creatAllIndex fail!"<clear(); + Q_EMIT this->transactionFinished(); + + return true; +} + +IndexGenerator::IndexGenerator(QObject *parent) : QObject(parent) +{ + m_datebase = new Xapian::WritableDatabase(INDEX_PATH, Xapian::DB_CREATE_OR_OPEN); + m_cryp = new QCryptographicHash(QCryptographicHash::Md5); +} + +IndexGenerator::~IndexGenerator() +{ +} + +void IndexGenerator::CreatIndex(QString *indexText,QString *doc) +{ + qDebug()<< "--index start--"; + m_docstr = doc->toStdString(); + m_index_text_str = indexText->toStdString(); + + std::string uniqueterm = m_cryp->hash(doc->toUtf8(),QCryptographicHash::Md5).toStdString(); + + Xapian::Document document; + document.set_data(m_docstr); + document.add_term(uniqueterm); + m_indexer->set_document(document); + m_indexer->index_text(m_index_text_str); + + Xapian::docid innerId= m_datebase->replace_document(uniqueterm,document); + + // qDebug()<<"replace doc uniqueterm="<; + QStringList *list = pathlist; + for(int i = 0;isize();i++) + { + auto info = new QFileInfo(list->at(i)); + //提取文件名并用空格分割,同时去除'.' + QString filename = info->fileName(); + QString index_test = filename.replace(".","").replace(""," "); + index_test.simplified(); + // qDebug()<<"index_test"<insert(info->absoluteFilePath(),index_test); + // qDebug()<value(index_test); + } + + qDebug()<<"Finish HandlePathList!"; + return; +} + +bool IndexGenerator::isIndexdataExist() +{ + +// Xapian::Database db(m_index_data_path->toStdString()); + + +} + +void IndexGenerator::IndexSearch(QString *indexText) +{ + try + { + qDebug()<<"--search start--"; + + Xapian::Enquire enquire(*m_datebase); + Xapian::QueryParser qp; + qp.set_default_op(Xapian::Query::OP_NEAR); + qp.set_database(*m_datebase); + + std::string queryStr = indexText->replace(""," ").toStdString(); +// std::string s =db.get_spelling_suggestion(queryStr,10); +// qDebug()<<"spelling_suggestion!"<searchFinish(); + return; +} + +bool IndexGenerator::deleteAllIndex(QStringList *pathlist) +{ + QStringList *list = pathlist; + if(list->isEmpty()) + return true; + for(int i = 0;isize();i++) + { + QString doc = list->at(i); + std::string uniqueterm = m_cryp->hash(doc.toUtf8(),QCryptographicHash::Md5).toStdString();; + try + { + qDebug()<<"--delete start--"; + m_datebase->delete_document(uniqueterm); + qDebug()<<"delete md5"<commit(); + qDebug()<< "--delete finish--"; + } + catch(const Xapian::Error &e) + { + qDebug() <transactionFinished(); + return true; +} + diff --git a/index/index-generator.h b/index/index-generator.h new file mode 100644 index 0000000..d15eb27 --- /dev/null +++ b/index/index-generator.h @@ -0,0 +1,42 @@ +#ifndef INDEXGENERATOR_H +#define INDEXGENERATOR_H + +#include +#include + +#include +#include +#include + +class IndexGenerator : public QObject +{ + Q_OBJECT +public: + static IndexGenerator *getInstance(); + bool isIndexdataExist(); +Q_SIGNALS: + void transactionFinished(); + void searchFinish(); +public Q_SLOTS: + bool creatAllIndex(QStringList *pathlist); + bool deleteAllIndex(QStringList *pathlist); + void IndexSearch(QString *indexText); +private: + explicit IndexGenerator(QObject *parent = nullptr); + void HandlePathList(QStringList *pathlist); + //add one data in database + void CreatIndex(QString *indexText,QString *doc); + ~IndexGenerator(); + + QMap *m_index_map; + QCryptographicHash *m_cryp; + QString *m_index_data_path; + Xapian::WritableDatabase *m_datebase; + std::string m_docstr; + std::string m_index_text_str; + Xapian::TermGenerator *m_indexer; + + +}; + +#endif // INDEXGENERATOR_H diff --git a/index/index.pri b/index/index.pri new file mode 100644 index 0000000..bf4c239 --- /dev/null +++ b/index/index.pri @@ -0,0 +1,13 @@ +INCLUDEPATH += $$PWD + +HEADERS += \ + $$PWD/index-generator.h \ + $$PWD/inotify-manager.h \ + $$PWD/messagelistmanager.h \ + +SOURCES += \ + $$PWD/index-generator.cpp \ + $$PWD/inotify-manager.cpp \ + $$PWD/messagelistmanager.cpp \ + $$PWD/testInotifyManager.cpp \ + diff --git a/index/inotify-manager.cpp b/index/inotify-manager.cpp new file mode 100644 index 0000000..8c494c7 --- /dev/null +++ b/index/inotify-manager.cpp @@ -0,0 +1,219 @@ +#include "inotify-manager.h" +#include "index-generator.h" +#include "messagelistmanager.h" + +bool InotifyManager::Traverse_BFS(const QString& path, const bool& CREATORDELETE){ + qDebug() << "BFS start-----------------------------"; + int total = 0; + MessageListManager mlm; + mlm.SetAutoSendMessageLength(80000); + QQueue bfs; + bfs.enqueue(path); + QFileInfoList list; + QDir dir; + dir.setFilter(QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot); + dir.setSorting(QDir::DirsFirst); + while (!bfs.empty()) { + dir.setPath(bfs.dequeue()); + list = dir.entryInfoList(); + for (auto i : list){ +// qDebug() << i.absoluteFilePath(); + if (i.isDir()){ + AddWatch(i.absoluteFilePath()); + bfs.enqueue(i.absoluteFilePath()); + } + else{ + mlm.AddMessage(i.absoluteFilePath()); + total++; + //continue; + } + } + } + mlm.SendMessage(); + qDebug()<creatAllIndex(new QStringList(fileInfo.filePath())); + } + } + i++; + } while (i < list.size()); + return true; +} + +bool InotifyManager::AddWatch(const QString &path){ + //m_fd = inotify_init(); +// qDebug() << "m_fd: " <" <::Iterator i = currentPath.begin(); i != currentPath.end();){ + if (i.value().length() > path.length()){ + if (i.value().mid(0, path.length()) == path){ + qDebug() << i.value(); + /*--------------------------------*/ + //在此调用删除索引 + IndexGenerator::getInstance()->deleteAllIndex(new QStringList(path)); + /*--------------------------------*/ + currentPath.erase(i++); + } + else{ + i++; + } + } + else{ + i++; + } + } + qDebug() << path; + //这个貌似不用删，先mark一下 + //currentPath.remove(currentPath.key(path)); + return true; +} + +void InotifyManager::run(){ + + char * p; + char buf[BUF_LEN] __attribute__((aligned(8))); + + ssize_t numRead; + + for (;;) { /* Read events forever */ + numRead = read(m_fd, buf, BUF_LEN); + if (numRead == 0) { + qDebug() << "read() from inotify fd returned 0!"; + } + if (numRead == -1) { + qDebug() << "read"; + } + qDebug() << "Read " << numRead << " bytes from inotify fd"; + + /* Process all of the events in buffer returned by read() */ + + for (p = buf; p < buf + numRead;) { + struct inotify_event * event = reinterpret_cast(p); + if(event->name[0] != '.'){ +// if(true){ + //这个位运算不要在意，只是懒得把文件夹、文件和事件排列组合了,只是看一下事件的类型 + qDebug() << "Read Event: " << num2string[(event->mask & 0x0000ffff)] << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd; + //num2string[event->mask & 0x0000ffff] + IndexGenerator::getInstance()->creatAllIndex(new QStringList(currentPath[event->wd] + event->name)); + + /*--------------------------------*/ + + //传创建或移动过来的文件路径 + if((event->mask & IN_CREATE) | (event->mask & IN_MOVED_TO)){ + //添加监视要先序遍历，先添加top节点 + if (event->mask & IN_ISDIR){ + AddWatch(currentPath[event->wd] + '/' + event->name); + Traverse_BFS(currentPath[event->wd] + '/' + event->name, true); + } + else { + //IndexGenerator::getInstance()->creatAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); + } + } + else if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){ + if (event->mask & IN_ISDIR){ + RemoveWatch(currentPath[event->wd] + '/' + event->name); + } + else { + //这里调用删除索引 + IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name)); + } + } + /*--------------------------------*/ + } + p += sizeof(struct inotify_event) + event->len; + } + } +} + + +InotifyManager::InotifyManager() +{ + + m_fd = inotify_init(); + qDebug() << "m_fd----------->" < +#include +#include +#include +#include +#include +#include +//#define EVENT_NUM 12 +#define BUF_LEN 1024 + + +class InotifyManager : public QThread +{ + Q_OBJECT +public: + explicit InotifyManager(); + bool Traverse(const QString&, const bool&);//true->create, false->delete + bool Traverse_BFS(const QString&, const bool&); + //typedef bool (*AddWatch)(const QString&); + //AddWatch cmp; + + bool AddWatch(const QString&); + bool AddWatchList(const QStringList&); + bool RemoveWatch(const QString&); + +protected: + void run() override; +private: + QString *m_watch_path; + int m_fd; + QMap currentPath; + QMap num2string; +}; + +void testTraverse(void); + +#endif // INOTIFYMANAGER_H diff --git a/index/messagelistmanager.cpp b/index/messagelistmanager.cpp new file mode 100644 index 0000000..1ba6a33 --- /dev/null +++ b/index/messagelistmanager.cpp @@ -0,0 +1,35 @@ +#include "messagelistmanager.h" +#include +MessageListManager::MessageListManager(){ + this->messageList = new QStringList(); + this->ig = IndexGenerator::getInstance(); +} + +MessageListManager::~MessageListManager(){ + delete this->messageList; + //delete this->ig; + + this->messageList = nullptr; + this->ig = nullptr; +} + +void MessageListManager::AddMessage(const QString& path){ + this->messageList->append(path); + if (static_cast(this->messageList->length()) >= this->length){ + this->SendMessage(); + } +} + +bool MessageListManager::SendMessage(){ + if (this->messageList->empty()){ + return true; + } + + this->ig->creatAllIndex(this->messageList); + this->messageList->clear(); + return true; +} + +void MessageListManager::SetAutoSendMessageLength(const size_t& length){ + this->length = length; +} diff --git a/index/messagelistmanager.h b/index/messagelistmanager.h new file mode 100644 index 0000000..5385ca9 --- /dev/null +++ b/index/messagelistmanager.h @@ -0,0 +1,26 @@ +#ifndef MESSAGELISTMANAGER_H +#define MESSAGELISTMANAGER_H + +#include +#include "index-generator.h" + +class MessageListManager : public QObject +{ + Q_OBJECT +public: + explicit MessageListManager(); + ~MessageListManager(); + + void AddMessage(const QString&); + bool SendMessage(); + void SetAutoSendMessageLength(const size_t&); + +private: + QStringList* messageList; + size_t length = 0; + IndexGenerator* ig; +Q_SIGNALS: + +}; + +#endif // MESSAGELISTMANAGER_H diff --git a/index/testInotifyManager.cpp b/index/testInotifyManager.cpp new file mode 100644 index 0000000..ab5896b --- /dev/null +++ b/index/testInotifyManager.cpp @@ -0,0 +1,38 @@ +#include "mainwindow.h" +#include "inotify-manager.h" +#include +#include + +void testTraverse(void){ + + +// QStringList qsl; +// for (int i = 0; i < 4000; i++){ +// qsl.append(QString("%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1%1").arg(i)); +// } + +// for (int i = 0; i < 4000; i++){ +// qDebug() << i << " " << qsl.at(i); +// if (qsl.at(i) != QString("%1").arg(i)){ +// qDebug() << "fuck stringlist"; +// } +// } + +// qDebug() << qsl.length(); +// qDebug() << "stringlist"; +// exit(0); + + + + + QTime t1 = QTime::currentTime(); + InotifyManager* im = new InotifyManager(); + im->AddWatch("/home/zpf"); + im->Traverse_BFS("/home/zpf", true); + QTime t2 = QTime::currentTime(); + qDebug() << t1; + qDebug() << t2; + im->start(); + + //exit(0); +} diff --git a/model/model.pri b/model/model.pri new file mode 100644 index 0000000..aba082d --- /dev/null +++ b/model/model.pri @@ -0,0 +1 @@ +INCLUDEPATH += $$PWD diff --git a/src/src.pri b/src/src.pri new file mode 100644 index 0000000..31e855f --- /dev/null +++ b/src/src.pri @@ -0,0 +1,10 @@ +INCLUDEPATH += $$PWD + +HEADERS += \ + $$PWD/mainwindow.h \ + +SOURCES += \ + $$PWD/main.cpp \ + $$PWD/mainwindow.cpp \ + + diff --git a/ukui-search.pro b/ukui-search.pro index 46fb397..0b51ab5 100644 --- a/ukui-search.pro +++ b/ukui-search.pro @@ -2,7 +2,7 @@ QT += core gui greaterThan(QT_MAJOR_VERSION, 4): QT += widgets -CONFIG += c++11 +CONFIG += c++11 no_keywords # The following define makes your compiler emit warnings if you use # any Qt feature that has been marked deprecated (the exact warnings @@ -15,14 +15,19 @@ DEFINES += QT_DEPRECATED_WARNINGS # You can also select to disable deprecated APIs only up to a certain version of Qt. #DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000 # disables all the APIs deprecated before Qt 6.0.0 -SOURCES += \ - main.cpp \ - mainwindow.cpp - -HEADERS += \ - mainwindow.h +include(src/src.pri)) +include(index/index.pri) +include(model/model.pri) +include(control/control.pri)) +LIBS = -lxapian # Default rules for deployment. qnx: target.path = /tmp/$${TARGET}/bin else: unix:!android: target.path = /opt/$${TARGET}/bin !isEmpty(target.path): INSTALLS += target + +HEADERS += \ + file-utils.h + +SOURCES += \ + file-utils.cpp