openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Merge branch '0605-dev' into 'dev-unity' 

Add inotify events queue for merging events.

See merge request kylin-desktop/ukui-search!31
This commit is contained in:
Zihao Zhang 2021-06-15 07:19:25 +00:00
parent 00365d291c d4939e118d
commit 8886ee098a
16 changed files with 1022 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
libsearch/common.h
View File

@ -1,7 +1,22 @@
#ifndef COMMON_H
#define COMMON_H
#include <QMap>
#define UKUI_SEARCH_PIPE_PATH (QDir::homePath()+"/.config/org.ukui/ukui-search/ukuisearch").toLocal8Bit().constData()
#define HOME_PATH QDir::homePath()
static const QMap<QString, bool> targetFileTypeMap = {
std::map<QString, bool>::value_type("doc", true),
std::map<QString, bool>::value_type("docx", true),
std::map<QString, bool>::value_type("ppt", true),
std::map<QString, bool>::value_type("pptx", true),
std::map<QString, bool>::value_type("xls", true),
std::map<QString, bool>::value_type("xlsx", true),
std::map<QString, bool>::value_type("txt", true),
std::map<QString, bool>::value_type("dot", true),
std::map<QString, bool>::value_type("wps", true),
std::map<QString, bool>::value_type("pps", true),
std::map<QString, bool>::value_type("dps", true),
std::map<QString, bool>::value_type("et", true),
std::map<QString, bool>::value_type("pdf", true)
};
//TODO Put things that needed to be put here here.
#endif // COMMON_H

2
libsearch/file-utils.cpp
View File

@ -516,7 +516,7 @@ void FileUtils::getDocxTextContent(QString &path, QString &textcontent) {
QDomElement wr = wp.firstChildElement("w:r");
while(!wr.isNull()) {
QDomElement wt = wr.firstChildElement("w:t");
textcontent.append(wt.text().replace("\n", ""));
textcontent.append(wt.text().replace("\n", "")).replace("\r", " ");
if(textcontent.length() >= MAX_CONTENT_LENGTH / 3) {
file.close();
return;

21
libsearch/index/construct-document.cpp
View File

@ -34,8 +34,8 @@ ConstructDocumentForPath::ConstructDocumentForPath(QVector<QString> list) {
void ConstructDocumentForPath::run() {
// qDebug()<<"ConstructDocumentForPath";
if(!Zeeker::_doc_list_path)
Zeeker::_doc_list_path = new QList<Document>;
// if(!Zeeker::_doc_list_path)
// Zeeker::_doc_list_path = new QVector<Document>;
// qDebug()<<_doc_list_path->size();
QString index_text = m_list.at(0).toLower();
QString sourcePath = m_list.at(1);
@ -87,9 +87,9 @@ void ConstructDocumentForPath::run() {
}
// QMetaObject::invokeMethod(m_indexGenerator,"appendDocListPath",Q_ARG(Document,doc));
Zeeker::_mutex_doc_list_path.lock();
Zeeker::_doc_list_path->append(doc);
Zeeker::_mutex_doc_list_path.unlock();
IndexGenerator::_mutex_doc_list_path.lock();
IndexGenerator::_doc_list_path.append(doc);
IndexGenerator::_mutex_doc_list_path.unlock();
// qDebug()<<"ConstructDocumentForPath finish";
return;
}
@ -102,8 +102,8 @@ ConstructDocumentForContent::ConstructDocumentForContent(QString path) {
void ConstructDocumentForContent::run() {
// qDebug() << "ConstructDocumentForContent currentThreadId()" << QThread::currentThreadId();
// 构造文本索引的document
if(!Zeeker::_doc_list_content)
Zeeker::_doc_list_content = new QList<Document>;
// if(!Zeeker::_doc_list_content)
// Zeeker::_doc_list_content = new QVector<Document>;
QString content;
FileReader::getTextContent(m_path, content);
if(content.isEmpty())
@ -126,11 +126,12 @@ void ConstructDocumentForContent::run() {
doc.addPosting(term.at(i).word, term.at(i).offsets, static_cast<int>(term.at(i).weight));
}
Zeeker::_mutex_doc_list_content.lock();
Zeeker::_doc_list_content->append(doc);
Zeeker::_mutex_doc_list_content.unlock();
IndexGenerator::_mutex_doc_list_content.lock();
IndexGenerator::_doc_list_content.append(doc);
IndexGenerator::_mutex_doc_list_content.unlock();
content.clear();
content.squeeze();
term.clear();
term.shrink_to_fit();
return;
}

190
libsearch/index/index-generator.cpp
View File

@ -28,7 +28,6 @@
#include "file-utils.h"
#include "index-generator.h"
#include "chinese-segmentation.h"
#include "construct-document.h"
#include <QStandardPaths>
@ -39,10 +38,14 @@ using namespace Zeeker;
static IndexGenerator *global_instance = nullptr;
QMutex IndexGenerator::m_mutex;
QList<Document> *Zeeker::_doc_list_path;
QMutex Zeeker::_mutex_doc_list_path;
QList<Document> *Zeeker::_doc_list_content;
QMutex Zeeker::_mutex_doc_list_content;
//QVector<Document> *Zeeker::_doc_list_path;
//QMutex Zeeker::_mutex_doc_list_path;
//QVector<Document> *Zeeker::_doc_list_content;
//QMutex Zeeker::_mutex_doc_list_content;
QMutex IndexGenerator::_mutex_doc_list_path;
QMutex IndexGenerator::_mutex_doc_list_content;
QVector<Document> IndexGenerator::_doc_list_path = QVector<Document>();
QVector<Document> IndexGenerator::_doc_list_content = QVector<Document>();
IndexGenerator *IndexGenerator::getInstance(bool rebuild, QObject *parent) {
QMutexLocker locker(&m_mutex);
@ -61,45 +64,33 @@ bool IndexGenerator::setIndexdataPath() {
//文件名索引
bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList) {
// FileUtils::_index_status |= 0x1;
// qDebug() << messageList->size();
HandlePathList(messageList);
if(_doc_list_path == NULL) {
// if(_doc_list_path == NULL) {
// return false;
// }
if(IndexGenerator::_doc_list_path.isEmpty()) {
return false;
}
qDebug() << "begin creatAllIndex";
// GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "0");
try {
// m_indexer = new Xapian::TermGenerator();
// m_indexer.set_database(*m_database_path);
//可以实现拼写纠正
// m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING);
// m_indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
// int count =0;
for(auto i : *_doc_list_path) {
for(auto i : IndexGenerator::_doc_list_path) {
insertIntoDatabase(i);
// if(++count > 8999){
// count = 0;
// m_database_path->commit();
// }
}
m_database_path->commit();
} catch(const Xapian::Error &e) {
qWarning() << "creatAllIndex fail!" << QString::fromStdString(e.get_description());
//need a record
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1");
// FileUtils::_index_status &= ~0x1;
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1");
assert(false);
}
// GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2");
qDebug() << "finish creatAllIndex";
// FileUtils::_index_status &= ~0x1;
_doc_list_path->clear();
delete _doc_list_path;
_doc_list_path = nullptr;
IndexGenerator::_doc_list_path.clear();
IndexGenerator::_doc_list_path.squeeze();
QVector<Document>().swap(IndexGenerator::_doc_list_path);
// delete _doc_list_path;
// _doc_list_path = nullptr;
return true;
}
//文件内容索引
@ -107,16 +98,19 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList) {
// FileUtils::_index_status |= 0x2;
HandlePathList(messageList);
qDebug() << "begin creatAllIndex for content";
if(_doc_list_content == NULL) {
// if(_doc_list_content == NULL) {
// return false;
// }
if(IndexGenerator::_doc_list_content.isEmpty()) {
return false;
}
int size = _doc_list_content->size();
int size = IndexGenerator::_doc_list_content.size();
qDebug() << "begin creatAllIndex for content" << size;
if(!size == 0) {
// GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "0");
try {
int count = 0;
for(auto i : *_doc_list_content) {
for(auto i : IndexGenerator::_doc_list_content) {
insertIntoContentDatabase(i);
if(++count > 999) {
count = 0;
@ -133,9 +127,11 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList) {
// GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2");
// FileUtils::_index_status &= ~0x2;
qDebug() << "finish creatAllIndex for content";
_doc_list_content->clear();
delete _doc_list_content;
_doc_list_content = nullptr;
IndexGenerator::_doc_list_content.clear();
IndexGenerator::_doc_list_content.squeeze();
QVector<Document>().swap(IndexGenerator::_doc_list_content);
// delete _doc_list_content;
// _doc_list_content = nullptr;
}
Q_EMIT this->transactionFinished();
return true;
@ -297,7 +293,7 @@ void IndexGenerator::HandlePathList(QQueue<QString> *messageList) {
return;
}
//deprecated
Document IndexGenerator::GenerateDocument(const QVector<QString> &list) {
Document doc;
// qDebug()<<QString::number(quintptr(QThread::currentThreadId()));
@ -342,7 +338,7 @@ Document IndexGenerator::GenerateDocument(const QVector<QString> &list) {
return doc;
}
//deprecated
Document IndexGenerator::GenerateContentDocument(const QString &path) {
// 构造文本索引的document
QString content;
@ -389,7 +385,7 @@ bool IndexGenerator::isIndexdataExist() {
}
//deprecated
QStringList IndexGenerator::IndexSearch(QString indexText) {
QStringList searchResult;
try {
@ -455,96 +451,60 @@ QStringList IndexGenerator::IndexSearch(QString indexText) {
return searchResult;
}
//void IndexGenerator::setSynonym()
//{
// try
// {
// m_database_path->add_synonym("a","A");
// m_database_path->add_synonym("b","B");
// m_database_path->add_synonym("c","C");
// m_database_path->add_synonym("d","D");
// m_database_path->add_synonym("e","A");
// m_database_path->add_synonym("f","F");
// m_database_path->add_synonym("g","G");
// m_database_path->add_synonym("h","H");
// m_database_path->add_synonym("i","I");
// m_database_path->add_synonym("j","J");
// m_database_path->add_synonym("k","K");
// m_database_path->add_synonym("l","L");
// m_database_path->add_synonym("m","M");
// m_database_path->add_synonym("n","N");
// m_database_path->add_synonym("o","O");
// m_database_path->add_synonym("p","P");
// m_database_path->add_synonym("q","Q");
// m_database_path->add_synonym("r","R");
// m_database_path->add_synonym("s","S");
// m_database_path->add_synonym("t","T");
// m_database_path->add_synonym("u","U");
// m_database_path->add_synonym("v","V");
// m_database_path->add_synonym("w","W");
// m_database_path->add_synonym("x","X");
// m_database_path->add_synonym("y","Y");
// m_database_path->add_synonym("z","Z");
// m_database_path->add_synonym("A","a");
// m_database_path->add_synonym("B","b");
// m_database_path->add_synonym("C","c");
// m_database_path->add_synonym("D","d");
// m_database_path->add_synonym("E","e");
// m_database_path->add_synonym("F","f");
// m_database_path->add_synonym("G","g");
// m_database_path->add_synonym("H","h");
// m_database_path->add_synonym("I","i");
// m_database_path->add_synonym("J","j");
// m_database_path->add_synonym("K","k");
// m_database_path->add_synonym("L","a");
// m_database_path->add_synonym("M","m");
// m_database_path->add_synonym("N","n");
// m_database_path->add_synonym("O","o");
// m_database_path->add_synonym("P","p");
// m_database_path->add_synonym("Q","q");
// m_database_path->add_synonym("R","r");
// m_database_path->add_synonym("S","s");
// m_database_path->add_synonym("T","t");
// m_database_path->add_synonym("U","u");
// m_database_path->add_synonym("V","v");
// m_database_path->add_synonym("W","w");
// m_database_path->add_synonym("X","x");
// m_database_path->add_synonym("Y","y");
// m_database_path->add_synonym("Z","z");
// m_database_path->commit();
// }
// catch(const Xapian::Error &e)
// {
// qWarning() <<QString::fromStdString(e.get_description());
// }
//}
bool IndexGenerator::deleteAllIndex(QStringList *pathlist) {
QStringList *list = pathlist;
if(list->isEmpty())
return true;
for(int i = 0; i < list->size(); i++) {
QString doc = list->at(i);
std::string uniqueterm = FileUtils::makeDocUterm(doc);
try {
try {
for(int i = 0; i < list->size(); i++) {
QString doc = list->at(i);
std::string uniqueterm = FileUtils::makeDocUterm(doc);
qDebug() << "--delete start--";
m_database_path->delete_document(uniqueterm);
m_database_content->delete_document(uniqueterm);
qDebug() << "delete path" << doc;
qDebug() << "delete md5" << QString::fromStdString(uniqueterm);
m_database_path->commit();
m_database_content->commit();
qDebug() << "--delete finish--";
// qDebug()<<"m_database_path->get_lastdocid()!!!"<<m_database_path->get_lastdocid();
// qDebug()<<"m_database_path->get_doccount()!!!"<<m_database_path->get_doccount();
} catch(const Xapian::Error &e) {
qWarning() << QString::fromStdString(e.get_description());
return false;
// qDebug()<<"m_database_path->get_lastdocid()!!!"<<m_database_path->get_lastdocid();
// qDebug()<<"m_database_path->get_doccount()!!!"<<m_database_path->get_doccount();
}
m_database_path->commit();
m_database_content->commit();
} catch(const Xapian::Error &e) {
qWarning() << QString::fromStdString(e.get_description());
return false;
}
Q_EMIT this->transactionFinished();
return true;
}
bool IndexGenerator::updateIndex(QVector<PendingFile> *pendingFiles)
{
QQueue<QVector<QString>> *fileIndexInfo = new QQueue<QVector<QString>>;
QQueue<QString> *fileContentIndexInfo = new QQueue<QString>;
QStringList *deleteList = new QStringList;
for(PendingFile file : *pendingFiles) {
if(file.shouldRemoveIndex()) {
deleteList->append(file.path());
continue;
}
fileIndexInfo->append(QVector<QString>() << file.path().section("/" , -1) << file.path() << QString(file.isDir() ? "1" : "0"));
if((!file.path().split(".").isEmpty()) && (true == targetFileTypeMap[file.path().section("/" , -1) .split(".").last()]))
fileContentIndexInfo->append(file.path());
}
if(!deleteList->isEmpty()) {
deleteAllIndex(deleteList);
}
if(!fileIndexInfo->isEmpty()) {
creatAllIndex(fileIndexInfo);
}
if(!fileContentIndexInfo->isEmpty()) {
creatAllIndex(fileContentIndexInfo);
}
delete fileIndexInfo;
delete fileContentIndexInfo;
return true;
}

19
libsearch/index/index-generator.h
View File

@ -29,18 +29,22 @@
#include <QMutex>
#include <QQueue>
//#include <QMetaObject>
#include "construct-document.h"
#include "index-status-recorder.h"
#include "document.h"
#include "file-reader.h"
#include "common.h"
#include "pending-file.h"
namespace Zeeker {
extern QList<Document> *_doc_list_path;
extern QMutex _mutex_doc_list_path;
extern QList<Document> *_doc_list_content;
extern QMutex _mutex_doc_list_content;
//extern QVector<Document> *_doc_list_path;
//extern QMutex _mutex_doc_list_path;
//extern QVector<Document> *_doc_list_content;
//extern QMutex _mutex_doc_list_content;
class IndexGenerator : public QObject {
friend class ConstructDocumentForPath;
friend class ConstructDocumentForContent;
Q_OBJECT
public:
static IndexGenerator *getInstance(bool rebuild = false, QObject *parent = nullptr);
@ -58,6 +62,7 @@ public Q_SLOTS:
bool creatAllIndex(QQueue<QVector<QString>> *messageList);
bool creatAllIndex(QQueue<QString> *messageList);
bool deleteAllIndex(QStringList *pathlist);
bool updateIndex(QVector<PendingFile> *pendingFiles);
private:
explicit IndexGenerator(bool rebuild = false, QObject *parent = nullptr);
@ -72,8 +77,10 @@ private:
void insertIntoDatabase(Document& doc);
void insertIntoContentDatabase(Document& doc);
// QList<Document> *m_doc_list_path; //for path index
// QList<Document> *m_doc_list_content; // for text content index
static QVector<Document> _doc_list_path;
static QMutex _mutex_doc_list_path;
static QVector<Document> _doc_list_content;
static QMutex _mutex_doc_list_content;
QMap<QString, QStringList> m_index_map;
QString m_index_data_path;
Xapian::WritableDatabase* m_database_path;

2
libsearch/index/index-status-recorder.cpp
View File

@ -13,8 +13,10 @@ IndexStatusRecorder *IndexStatusRecorder::getInstance()
void IndexStatusRecorder::setStatus(const QString &key, const QVariant &value)
{
m_mutex.lock();
m_status->setValue(key, value);
m_status->sync();
m_mutex.unlock();
}
const QVariant IndexStatusRecorder::getStatus(const QString &key)

3
libsearch/index/index-status-recorder.h
View File

@ -4,9 +4,11 @@
#include <QObject>
#include <QSettings>
#include <QDir>
#include <QMutex>
#define CONTENT_INDEX_DATABASE_STATE "content_index_database_state"
#define INDEX_DATABASE_STATE "index_database_state"
#define INOTIFY_NORMAL_EXIT "inotify_normal_exit"
#define PENDING_FILE_QUEUE_FINISH "pending_file_queue_finish"
#define INDEX_STATUS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-index-status.conf"
namespace Zeeker {
//fixme: we need a better way to record index status.
@ -21,6 +23,7 @@ public:
private:
explicit IndexStatusRecorder(QObject *parent = nullptr);
QSettings *m_status;
QMutex m_mutex;
};
}

6
libsearch/index/index.pri
View File

@ -8,6 +8,9 @@ HEADERS += \
$$PWD/index-generator.h \
$$PWD/index-status-recorder.h \
$$PWD/inotify-index.h \
$$PWD/inotify-watch.h \
$$PWD/pending-file-queue.h \
$$PWD/pending-file.h \
$$PWD/search-manager.h \
$$PWD/searchmethodmanager.h \
$$PWD/traverse_bfs.h \
@ -21,6 +24,9 @@ SOURCES += \
$$PWD/index-generator.cpp \
$$PWD/index-status-recorder.cpp \
$$PWD/inotify-index.cpp \
$$PWD/inotify-watch.cpp \
$$PWD/pending-file-queue.cpp \
$$PWD/pending-file.cpp \
$$PWD/search-manager.cpp \
$$PWD/searchmethodmanager.cpp \
$$PWD/traverse_bfs.cpp \

445
libsearch/index/inotify-watch.cpp Normal file
View File

@ -0,0 +1,445 @@
#include "inotify-watch.h"
#include <sys/ioctl.h>
#include <malloc.h>
using namespace Zeeker;
static InotifyWatch* global_instance_InotifyWatch = nullptr;
Zeeker::InotifyWatch *Zeeker::InotifyWatch::getInstance(const QString &path)
{
if(!global_instance_InotifyWatch) {
global_instance_InotifyWatch = new InotifyWatch(path);
}
return global_instance_InotifyWatch;
}
Zeeker::InotifyWatch::InotifyWatch(const QString &path): Traverse_BFS(path)
{
qDebug() << "setInotifyMaxUserWatches start";
UkuiSearchQDBus usQDBus;
usQDBus.setInotifyMaxUserWatches();
qDebug() << "setInotifyMaxUserWatches end";
m_sharedMemory = new QSharedMemory("ukui-search-shared-map", this);
}
InotifyWatch::~InotifyWatch()
{
if(m_notifier)
delete m_notifier;
m_notifier = nullptr;
}
bool InotifyWatch::addWatch(const QString &path)
{
int ret = inotify_add_watch(m_inotifyFd, path.toStdString().c_str(), (IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | IN_DELETE | IN_MODIFY));
if(ret == -1) {
qWarning() << "AddWatch error:" << path;
return false;
}
currentPath[ret] = path;
// qDebug() << "Watch: " << path << "ret: " << ret;
return true;
}
bool InotifyWatch::removeWatch(const QString &path, bool removeFromDatabase)
{
inotify_rm_watch(m_inotifyFd, currentPath.key(path));
if(removeFromDatabase) {
for(QMap<int, QString>::Iterator i = currentPath.begin(); i != currentPath.end();) {
// qDebug() << i.value();
// if(i.value().length() > path.length()) {
if(i.value().startsWith(path)) {
qDebug() << "remove path: " << i.value();
inotify_rm_watch(m_inotifyFd, currentPath.key(path));
PendingFile f(i.value());
f.setDeleted();
f.setIsDir();
PendingFileQueue::getInstance()->enqueue(f);
currentPath.erase(i++);
} else {
i++;
}
}
} else {
for(QMap<int, QString>::Iterator i = currentPath.begin(); i != currentPath.end();) {
// qDebug() << i.value();
if(i.value().length() > path.length()) {
if(i.value().startsWith(path)) {
qDebug() << "remove path: " << i.value();
inotify_rm_watch(m_inotifyFd, currentPath.key(path));
currentPath.erase(i++);
} else {
i++;
}
} else {
i++;
}
}
}
currentPath.remove(currentPath.key(path));
return true;
}
void InotifyWatch::DoSomething(const QFileInfo &info)
{
qDebug() << info.fileName() << "-------" << info.absoluteFilePath();
if(info.isDir() && (!info.isSymLink())) {
this->addWatch(info.absoluteFilePath());
}
PendingFile f(info.absoluteFilePath());
if(info.isDir()) {
f.setIsDir();
}
PendingFileQueue::getInstance()->enqueue(f);
}
void InotifyWatch::firstTraverse()
{
QQueue<QString> bfs;
bfs.enqueue(this->path);
QFileInfoList list;
QDir dir;
dir.setFilter(QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot);
dir.setSorting(QDir::DirsFirst);
while(!bfs.empty()) {
dir.setPath(bfs.dequeue());
list = dir.entryInfoList();
for(auto i : list) {
if(i.isDir() && (!(i.isSymLink()))) {
this->addWatch(i.absoluteFilePath());
bfs.enqueue(i.absoluteFilePath());
}
}
}
}
void InotifyWatch::stopWatch()
{
// if(this->isRunning()) {
// this->quit();
// if(m_notifier)
// delete m_notifier;
// m_notifier = nullptr;
// removeWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation), false);
// }
// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "3");
}
void InotifyWatch::run()
{
m_inotifyFd = inotify_init();
if (m_inotifyFd > 0) {
qDebug()<<"Inotify init success!";
} else {
Q_ASSERT_X(0, "InotifyWatch", "Failed to initialize inotify");
}
this->addWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation));
this->setPath(QStandardPaths::writableLocation(QStandardPaths::HomeLocation));
this->firstTraverse();
int fifo_fd;
char buffer[2];
memset(buffer, 0, sizeof(buffer));
fifo_fd = open(UKUI_SEARCH_PIPE_PATH, O_RDWR);
if(fifo_fd == -1) {
qWarning() << "Open fifo error\n";
assert(false);
}
int retval = read(fifo_fd, buffer, sizeof(buffer));
if(retval == -1) {
qWarning() << "read error\n";
assert(false);
}
qDebug("Read fifo[%s]", buffer);
qDebug("Read data ok");
close(fifo_fd);
if(buffer[0] & 0x1) {
qDebug("Data confirmed\n");
}
unlink(UKUI_SEARCH_PIPE_PATH);
while(FileUtils::SearchMethod::INDEXSEARCH == FileUtils::searchMethod) {
fd_set fds;
FD_ZERO(&fds);
FD_SET(m_inotifyFd, &fds);
int rc;
rc = select(m_inotifyFd + 1, &fds, NULL, NULL, NULL);
if(rc > 0) {
int avail;
if (ioctl(m_inotifyFd, FIONREAD, &avail) == EINVAL) {
qWarning() << "Did not receive an entire inotify event.";
return;
}
char* buf = (char*)malloc(avail);
memset(buf, 0x00, avail);
const ssize_t len = read(m_inotifyFd, buf, avail);
if(len != avail) {
qWarning()<<"read event error";
// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
}
int i = 0;
while (i < len) {
const struct inotify_event* event = (struct inotify_event*)&buf[i];
if(event->name[0] != '.') {
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
// qDebug("mask:0x%x,",event->mask);
break;
}
i += sizeof(struct inotify_event) + event->len;
}
if(i < len ) {
qDebug() << "fork";
slotEvent(buf, len);
free(buf);
}
} else if(rc < 0) {
// error
qWarning() << "select result < 0, error!";
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
assert(false);
}
}
if(FileUtils::SearchMethod::DIRECTSEARCH == FileUtils::searchMethod) {
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "3");
removeWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation), false);
}
// fcntl(m_inotifyFd, F_SETFD, FD_CLOEXEC);
// m_notifier = new QSocketNotifier(m_inotifyFd, QSocketNotifier::Read);
// connect(m_notifier, &QSocketNotifier::activated, this, &InotifyWatch::slotEvent, Qt::DirectConnection);
// exec();
}
void InotifyWatch::slotEvent(char *buf, ssize_t len)
{
// eventProcess(socket);
++FileUtils::_index_status;
if(FileUtils::SearchMethod::INDEXSEARCH == FileUtils::searchMethod) {
pid_t pid;
pid = fork();
if(pid == 0) {
prctl(PR_SET_PDEATHSIG, SIGTERM);
prctl(PR_SET_NAME, "inotify-index");
this->eventProcess(buf, len);
fd_set read_fds;
int rc;
timeval* read_timeout = (timeval*)malloc(sizeof(timeval));
read_timeout->tv_sec = 40;
read_timeout->tv_usec = 0;
for(;;) {
FD_ZERO(&read_fds);
FD_SET(m_inotifyFd, &read_fds);
rc = select(m_inotifyFd + 1, &read_fds, NULL, NULL, read_timeout);
if(rc < 0) {
// error
qWarning() << "fork select result < 0, error!";
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
assert(false);
} else if(rc == 0) {
qDebug() << "select timeout!";
::free(read_timeout);
QBuffer buffer;
QDataStream out(&buffer);
if (m_sharedMemory->isAttached()) {
m_sharedMemory->detach();
}
buffer.open(QBuffer::ReadWrite);
out << currentPath;
int size = buffer.size();
if (!m_sharedMemory->create(size)) {
qDebug() << "Create sharedMemory Error: " << m_sharedMemory->errorString();
} else {
m_sharedMemory->lock();
char *to = static_cast<char *>(m_sharedMemory->data());
const char *from = buffer.data().constData();
memcpy(to, from, qMin(size, m_sharedMemory->size()));
m_sharedMemory->unlock();
}
// GlobalSettings::getInstance()->forceSync();
PendingFileQueue::getInstance()->forceFinish();
PendingFileQueue::getInstance()->~PendingFileQueue();
::_exit(0);
} else {
// qDebug() << "Select remain:" <<read_timeout->tv_sec;
this->eventProcess(m_inotifyFd);
// qDebug() << "Select remain:" <<read_timeout->tv_sec;
}
}
} else if(pid > 0) {
waitpid(pid, NULL, 0);
if (!m_sharedMemory->attach()) {
qDebug() << "SharedMemory attach Error: " << m_sharedMemory->errorString();
} else {
QBuffer buffer;
QDataStream in(&buffer);
QMap<int, QString> pathMap;
m_sharedMemory->lock();
buffer.setData(static_cast<const char *>(m_sharedMemory->constData()), m_sharedMemory->size());
buffer.open(QBuffer::ReadWrite);
in >> pathMap;
m_sharedMemory->unlock();
m_sharedMemory->detach();
currentPath = pathMap;
}
--FileUtils::_index_status;
} else {
assert(false);
}
}
}
char * InotifyWatch::filter()
{
int avail;
if (ioctl(m_inotifyFd, FIONREAD, &avail) == EINVAL) {
qWarning() << "Did not receive an entire inotify event.";
return NULL;
}
char* buffer = (char*)malloc(avail);
memset(buffer, 0x00, avail);
const int len = read(m_inotifyFd, buffer, avail);
if(len != avail) {
qWarning()<<"read event error";
// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
}
int i = 0;
while (i < len) {
const struct inotify_event* event = (struct inotify_event*)&buffer[i];
if(event->name[0] == '.') {
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
// qDebug("mask:0x%x,",event->mask);
i += sizeof(struct inotify_event) + event->len;
return buffer;
}
}
free(buffer);
return NULL;
}
void InotifyWatch::eventProcess(int socket)
{
// qDebug()<< "Enter eventProcess!";
int avail;
if (ioctl(socket, FIONREAD, &avail) == EINVAL) {
qWarning() << "Did not receive an entire inotify event.";
return;
}
char* buffer = (char*)malloc(avail);
memset(buffer, 0x00, avail);
const ssize_t len = read(socket, buffer, avail);
if(len != avail) {
qWarning()<<"read event error";
// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
}
int i = 0;
while (i < len) {
const struct inotify_event* event = (struct inotify_event*)&buffer[i];
if(event->name[0] != '.') {
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
// qDebug("mask:0x%x,",event->mask);
break;
}
i += sizeof(struct inotify_event) + event->len;
}
if(i >= len) {
qDebug() << "There is nothing to do!";
return;
}
eventProcess(buffer, len);
free(buffer);
}
void InotifyWatch::eventProcess(const char *buffer, ssize_t len)
{
// qDebug()<< "Begin eventProcess! len:" << len;
char * p = const_cast<char*>(buffer);
while (p < buffer + len) {
const struct inotify_event* event = reinterpret_cast<inotify_event *>(p);
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
// qDebug("mask:0x%x,",event->mask);
if(event->name[0] != '.') {
QString path = currentPath[event->wd] + '/' + event->name;
//Create top dir first, traverse it last.
if(event->mask & IN_CREATE) {
// qDebug() << "IN_CREATE";
PendingFile f(path);
if(event->mask & IN_ISDIR) {
f.setIsDir();
}
PendingFileQueue::getInstance(this)->enqueue(f);
if(event->mask & IN_ISDIR) {
if(!QFileInfo(path).isSymLink()){
addWatch(path);
setPath(path);
Traverse();
}
}
goto next;
}
if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)) {
qDebug() << "IN_DELETE or IN_MOVED_FROM";
if(event->mask & IN_ISDIR) {
removeWatch(path);
} else {
PendingFile f(path);
f.setDeleted();
PendingFileQueue::getInstance()->enqueue(f);
}
p += sizeof(struct inotify_event) + event->len;
continue;
}
if(event->mask & IN_MODIFY) {
// qDebug() << "IN_MODIFY";
if(!(event->mask & IN_ISDIR)) {
PendingFileQueue::getInstance()->enqueue(PendingFile(path));
}
goto next;
}
if(event->mask & IN_MOVED_TO) {
qDebug() << "IN_MOVED_TO";
if(event->mask & IN_ISDIR) {
removeWatch(path);
PendingFile f(path);
f.setIsDir();
PendingFileQueue::getInstance()->enqueue(f);
if(!QFileInfo(path).isSymLink()){
addWatch(path);
setPath(path);
Traverse();
}
} else {
//Enqueue a deleted file to merge.
PendingFile f(path);
f.setDeleted();
PendingFileQueue::getInstance()->enqueue(f);
//Enqueue a new one.
PendingFileQueue::getInstance()->enqueue(PendingFile(path));
}
goto next;
}
}
next:
p += sizeof(struct inotify_event) + event->len;
}
// qDebug()<< "Finish eventProcess!";
}

52
libsearch/index/inotify-watch.h Normal file
View File

@ -0,0 +1,52 @@
#ifndef INOTIFYWATCH_H
#define INOTIFYWATCH_H
#include <QThread>
#include <unistd.h>
#include <sys/inotify.h>
#include <QSocketNotifier>
#include <QDataStream>
#include <QSharedMemory>
#include "traverse_bfs.h"
#include "ukui-search-qdbus.h"
#include "index-status-recorder.h"
#include "file-utils.h"
#include "first-index.h"
#include "pending-file-queue.h"
#include "common.h"
namespace Zeeker {
class InotifyWatch : public QThread, public Traverse_BFS
{
Q_OBJECT
public:
static InotifyWatch* getInstance(const QString& path);
bool addWatch(const QString &path);
bool removeWatch(const QString &path, bool removeFromDatabase = true);
virtual void DoSomething(const QFileInfo &info) final;
void firstTraverse();
void stopWatch();
protected:
void run() override;
private Q_SLOTS:
void slotEvent(char *buf, ssize_t len);
private:
explicit InotifyWatch(const QString& path);
~InotifyWatch();
char * filter();
void eventProcess(int socket);
void eventProcess(const char *buffer, ssize_t len);
int m_inotifyFd;
QSocketNotifier* m_notifier = nullptr;
QSharedMemory *m_sharedMemory = nullptr;
QMap<int, QString> currentPath;
QMutex m_mutex;
};
}
#endif // INOTIFYWATCH_H

165
libsearch/index/pending-file-queue.cpp Normal file
View File

@ -0,0 +1,165 @@
/*
* Copyright (C) 2021, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#include "pending-file-queue.h"
#include <malloc.h>
using namespace Zeeker;
static PendingFileQueue *global_instance_pending_file_queue = nullptr;
PendingFileQueue::PendingFileQueue(QObject *parent) : QThread(parent)
{
this->start();
m_cacheTimer = new QTimer;
m_minProcessTimer = new QTimer;
m_cacheTimer->setInterval(10*1000);
m_cacheTimer->setSingleShot(true);
m_minProcessTimer->setInterval(500);
m_minProcessTimer->setSingleShot(true);
m_cacheTimer->moveToThread(this);
m_minProcessTimer->moveToThread(this);
// connect(this, &PendingFileQueue::cacheTimerStart, m_cacheTimer, f, Qt::DirectConnection);
// connect(this, &PendingFileQueue::minProcessTimerStart, m_minProcessTimer, f,Qt::DirectConnection);
connect(this, SIGNAL(cacheTimerStart()), m_cacheTimer, SLOT(start()));
connect(this, SIGNAL(minProcessTimerStart()), m_minProcessTimer, SLOT(start()));
connect(this, &PendingFileQueue::timerStop, m_cacheTimer, &QTimer::stop);
connect(this, &PendingFileQueue::timerStop, m_minProcessTimer, &QTimer::stop);
connect(m_cacheTimer, &QTimer::timeout, this, &PendingFileQueue::processCache, Qt::DirectConnection);
connect(m_minProcessTimer, &QTimer::timeout, this, &PendingFileQueue::processCache, Qt::DirectConnection);
}
PendingFileQueue *PendingFileQueue::getInstance(QObject *parent)
{
if (!global_instance_pending_file_queue) {
global_instance_pending_file_queue = new PendingFileQueue(parent);
}
return global_instance_pending_file_queue;
}
PendingFileQueue::~PendingFileQueue()
{
if(m_cacheTimer) {
delete m_cacheTimer;
m_cacheTimer = nullptr;
}
if(m_minProcessTimer) {
delete m_minProcessTimer;
m_minProcessTimer = nullptr;
}
IndexGenerator::getInstance()->~IndexGenerator();
}
void PendingFileQueue::forceFinish()
{
QThread::msleep(600);
Q_EMIT timerStop();
this->quit();
this->wait();
}
void PendingFileQueue::enqueue(const PendingFile &file)
{
// qDebug() << "enqueuq file: " << file.path();
m_mutex.lock();
m_enqueuetimes++;
if(m_cache.isEmpty()) {
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "0");
}
// Remove all indexs of files under a dir which is to about be deleted,but keep delete signals.
// Because our datebase need to delete those indexs one by one.
if(file.shouldRemoveIndex() && file.isDir()) {
const auto keepFile = [&file](const PendingFile& pending) {
return (!pending.path().startsWith(file.path()) || pending.shouldRemoveIndex());
};
const auto end = m_cache.end();
const auto droppedFilesBegin = std::stable_partition(m_cache.begin(), end, keepFile);
m_cache.erase(droppedFilesBegin, end);
}
if(file.shouldRemoveIndex()) {
m_cache.removeOne(file);
}
int i = m_cache.indexOf(file);
if (i == -1) {
// qDebug() << "insert file" << file.path() << file.shouldRemoveIndex();
m_cache << file;
} else {
// qDebug() << "merge file" << file.path() << file.shouldRemoveIndex();
m_cache[i].merge(file);
}
if(!m_cacheTimer->isActive()) {
// qDebug()<<"m_cacheTimer-----start!!";
// m_cacheTimer->start();
Q_EMIT cacheTimerStart();
}
Q_EMIT minProcessTimerStart();
// m_minProcessTimer->start();
// qDebug()<<"m_minProcessTimer-----start!!";
m_mutex.unlock();
// qDebug() << "Current cache-------------";
// for(PendingFile i : m_cache) {
// qDebug() << "|" << i.path();
// qDebug() << "|" <<i.shouldRemoveIndex();
// }
// qDebug() << "Current cache-------------";
// qDebug()<<"enqueuq file finish!!"<<file.path();
}
void PendingFileQueue::run()
{
exec();
}
void PendingFileQueue::processCache()
{
qDebug()<< "Begin processCache!" ;
m_mutex.lock();
qDebug() << "Events:" << m_enqueuetimes;
m_enqueuetimes = 0;
m_cache.swap(m_pendingFiles);
// m_pendingFiles = m_cache;
// m_cache.clear();
// m_cache.squeeze();
m_mutex.unlock();
qDebug() << "Current process-------------";
for(PendingFile i : m_pendingFiles) {
qDebug() << "|" << i.path();
qDebug() << "|" <<i.shouldRemoveIndex();
}
qDebug() << "Current process-------------";
if(m_pendingFiles.isEmpty()) {
qDebug()<< "Empty, finish processCache!";
return;
}
IndexGenerator::getInstance()->updateIndex(&m_pendingFiles);
m_mutex.lock();
if(m_cache.isEmpty()) {
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2");
}
m_mutex.unlock();
m_pendingFiles.clear();
m_pendingFiles.squeeze();
malloc_trim(0);
qDebug()<< "Finish processCache!";
return;
}

67
libsearch/index/pending-file-queue.h Normal file
View File

@ -0,0 +1,67 @@
/*
* Copyright (C) 2021, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#ifndef PENDINGFILEQUEUE_H
#define PENDINGFILEQUEUE_H
#include <QObject>
#include <QVector>
#include <QTimer>
#include <QThread>
#include <QMutex>
#include "pending-file.h"
#include "index-generator.h"
namespace Zeeker {
class PendingFileQueue : public QThread
{
Q_OBJECT
public:
static PendingFileQueue *getInstance(QObject *parent = nullptr);
~PendingFileQueue();
//This method will block until current cache has been processed.
//Do not do enqueue operation in other thread while this method is running.
void forceFinish();
void enqueue(const PendingFile& file);
QTimer *m_cacheTimer = nullptr;
QTimer *m_minProcessTimer = nullptr;
protected:
void run() override;
Q_SIGNALS:
void cacheTimerStart();
void minProcessTimerStart();
void timerStop();
private:
void processCache();
explicit PendingFileQueue(QObject *parent = nullptr);
QVector<PendingFile> m_cache;
QVector<PendingFile> m_pendingFiles;
QMutex m_mutex;
QMutex m_timeoutMutex;
QThread *m_timerThread = nullptr;
bool m_timeout = false;
int m_enqueuetimes = 0;
};
}
#endif // PENDINGFILEQUEUE_H

81
libsearch/index/pending-file.cpp Normal file
View File

@ -0,0 +1,81 @@
/*
* Copyright (C) 2021, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#include "pending-file.h"
using namespace Zeeker;
PendingFile::PendingFile(const QString &path)
: m_path(path)
, m_deleted(false)
, m_modified(false)
, m_isDir(false)
{
}
QString PendingFile::path() const
{
return m_path;
}
void PendingFile::setPath(const QString& path)
{
if (path.endsWith(QLatin1Char('/'))) {
m_path = path.mid(0, m_path.length() - 1);
return;
}
m_path = path;
}
//bool PendingFile::isNewFile() const
//{
// return m_created;
//}
//bool PendingFile::shouldIndexContents() const
//{
// if (m_created || m_modified) {
// return true;
// }
// return false;
//}
bool PendingFile::isDir() const
{
return m_isDir;
}
bool PendingFile::shouldRemoveIndex() const
{
return m_deleted;
}
void PendingFile::merge(const PendingFile& file)
{
// m_created |= file.m_created;
m_modified = file.m_modified;
m_deleted = file.m_deleted;
}
void PendingFile::printFlags() const
{
// qDebug() << "Created:" << m_created;
qDebug() << "Deleted:" << m_deleted;
qDebug() << "Modified:" << m_modified;
qDebug() << "Is dir:" << m_isDir;
}

65
libsearch/index/pending-file.h Normal file
View File

@ -0,0 +1,65 @@
/*
* Copyright (C) 2021, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#ifndef PENDINGFILE_H
#define PENDINGFILE_H
#include <QString>
#include <QDebug>
namespace Zeeker {
/**
* Represents a file/folder which needs to be indexed.
*/
class PendingFile
{
public:
explicit PendingFile(const QString& path = QString());
QString path() const;
void setPath(const QString& path);
void setIsDir(){ m_isDir = true; }
void setModified() { m_modified = true; }
// void setCreated() { m_created = true; }
void setDeleted() { m_deleted = true; }
bool shouldRemoveIndex() const;
// bool shouldIndexContents() const;
bool isDir() const;
bool operator == (const PendingFile& rhs) const {
return (m_path == rhs.m_path);
}
/**
* Takes a PendingFile \p file and merges its flags into
* the current PendingFile
*/
void merge(const PendingFile& file);
private:
QString m_path;
// bool m_created : 1;
bool m_deleted : 1;
bool m_modified : 1;
bool m_isDir : 1;
void printFlags() const;
};
}
#endif // PENDINGFILE_H

19
libsearch/index/searchmethodmanager.cpp
View File

@ -1,25 +1,34 @@
#include "searchmethodmanager.h"
using namespace Zeeker;
SearchMethodManager::SearchMethodManager()
{
m_iw = InotifyWatch::getInstance(HOME_PATH);
}
void SearchMethodManager::searchMethod(FileUtils::SearchMethod sm) {
qWarning() << "searchMethod start: " << static_cast<int>(sm);
if(FileUtils::SearchMethod::INDEXSEARCH == sm || FileUtils::SearchMethod::DIRECTSEARCH == sm) {
FileUtils::searchMethod = sm;
} else {
printf("enum class error!!!\n");
qWarning("enum class error!!!\n");
}
if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::_index_status) {
qWarning() << "start first index";
// m_fi = FirstIndex("/home/zhangzihao/Desktop");
m_fi.start();
qWarning() << "start inotify index";
// InotifyIndex ii("/home");
// ii.start();
this->m_ii = InotifyIndex::getInstance("/home");
if(!this->m_ii->isRunning()) {
this->m_ii->start();
// this->m_ii = InotifyIndex::getInstance("/home");
// if(!this->m_ii->isRunning()) {
// this->m_ii->start();
// }
if(!this->m_iw->isRunning()) {
this->m_iw->start();
}
qDebug() << "Search method has been set to INDEXSEARCH";
}
if(FileUtils::SearchMethod::DIRECTSEARCH == sm) {
m_iw->stopWatch();
}
qWarning() << "searchMethod end: " << static_cast<int>(FileUtils::searchMethod);
}

8
libsearch/index/searchmethodmanager.h
View File

@ -2,15 +2,17 @@
#define SEARCHMETHODMANAGER_H
#include "first-index.h"
#include "inotify-index.h"
//#include "inotify-index.h"
#include "inotify-watch.h"
namespace Zeeker {
class SearchMethodManager {
public:
SearchMethodManager() = default;
SearchMethodManager();
void searchMethod(FileUtils::SearchMethod sm);
private:
FirstIndex m_fi;
InotifyIndex* m_ii;
// InotifyIndex* m_ii;
InotifyWatch *m_iw = nullptr;
};
}