openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Optimized pending file queue process logic.

This commit is contained in:
iaom 2021-06-11 19:58:32 +08:00
parent b9da7ad708
commit d4939e118d
4 changed files with 57 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
libsearch/index/index-generator.cpp
View File

@ -455,26 +455,26 @@ bool IndexGenerator::deleteAllIndex(QStringList *pathlist) {
QStringList *list = pathlist; QStringList *list = pathlist;
if(list->isEmpty()) if(list->isEmpty())
return true; return true;
for(int i = 0; i < list->size(); i++) { try {
QString doc = list->at(i); for(int i = 0; i < list->size(); i++) {
std::string uniqueterm = FileUtils::makeDocUterm(doc); QString doc = list->at(i);
try { std::string uniqueterm = FileUtils::makeDocUterm(doc);
qDebug() << "--delete start--"; qDebug() << "--delete start--";
m_database_path->delete_document(uniqueterm); m_database_path->delete_document(uniqueterm);
m_database_content->delete_document(uniqueterm); m_database_content->delete_document(uniqueterm);
qDebug() << "delete path" << doc; qDebug() << "delete path" << doc;
qDebug() << "delete md5" << QString::fromStdString(uniqueterm); qDebug() << "delete md5" << QString::fromStdString(uniqueterm);
m_database_path->commit();
m_database_content->commit();
qDebug() << "--delete finish--"; qDebug() << "--delete finish--";
// qDebug()<<"m_database_path->get_lastdocid()!!!"<<m_database_path->get_lastdocid(); // qDebug()<<"m_database_path->get_lastdocid()!!!"<<m_database_path->get_lastdocid();
// qDebug()<<"m_database_path->get_doccount()!!!"<<m_database_path->get_doccount();
// qDebug()<<"m_database_path->get_doccount()!!!"<<m_database_path->get_doccount();
} catch(const Xapian::Error &e) {
qWarning() << QString::fromStdString(e.get_description());
return false;
} }
m_database_path->commit();
m_database_content->commit();
} catch(const Xapian::Error &e) {
qWarning() << QString::fromStdString(e.get_description());
return false;
} }
Q_EMIT this->transactionFinished(); Q_EMIT this->transactionFinished();
return true; return true;
} }

28
libsearch/index/inotify-watch.cpp
View File

@ -198,6 +198,11 @@ void InotifyWatch::run()
slotEvent(buf, len); slotEvent(buf, len);
free(buf); free(buf);
} }
} else if(rc < 0) {
// error
qWarning() << "select result < 0, error!";
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
assert(false);
} }
} }
if(FileUtils::SearchMethod::DIRECTSEARCH == FileUtils::searchMethod) { if(FileUtils::SearchMethod::DIRECTSEARCH == FileUtils::searchMethod) {
@ -220,7 +225,6 @@ void InotifyWatch::slotEvent(char *buf, ssize_t len)
if(pid == 0) { if(pid == 0) {
prctl(PR_SET_PDEATHSIG, SIGTERM); prctl(PR_SET_PDEATHSIG, SIGTERM);
prctl(PR_SET_NAME, "inotify-index"); prctl(PR_SET_NAME, "inotify-index");
this->eventProcess(buf, len); this->eventProcess(buf, len);
fd_set read_fds; fd_set read_fds;
int rc; int rc;
@ -230,11 +234,10 @@ void InotifyWatch::slotEvent(char *buf, ssize_t len)
for(;;) { for(;;) {
FD_ZERO(&read_fds); FD_ZERO(&read_fds);
FD_SET(m_inotifyFd, &read_fds); FD_SET(m_inotifyFd, &read_fds);
qDebug() << read_timeout->tv_sec;
rc = select(m_inotifyFd + 1, &read_fds, NULL, NULL, read_timeout); rc = select(m_inotifyFd + 1, &read_fds, NULL, NULL, read_timeout);
if(rc < 0) { if(rc < 0) {
// error // error
qWarning() << "select result < 0, error!"; qWarning() << "fork select result < 0, error!";
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1"); IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
assert(false); assert(false);
} else if(rc == 0) { } else if(rc == 0) {
@ -263,9 +266,9 @@ void InotifyWatch::slotEvent(char *buf, ssize_t len)
PendingFileQueue::getInstance()->~PendingFileQueue(); PendingFileQueue::getInstance()->~PendingFileQueue();
::_exit(0); ::_exit(0);
} else { } else {
qDebug() << "Select remain:" <<read_timeout->tv_sec; // qDebug() << "Select remain:" <<read_timeout->tv_sec;
this->eventProcess(m_inotifyFd); this->eventProcess(m_inotifyFd);
qDebug() << "Select remain:" <<read_timeout->tv_sec; // qDebug() << "Select remain:" <<read_timeout->tv_sec;
} }
} }
} else if(pid > 0) { } else if(pid > 0) {
@ -323,7 +326,7 @@ char * InotifyWatch::filter()
} }
void InotifyWatch::eventProcess(int socket) void InotifyWatch::eventProcess(int socket)
{ {
qDebug()<< "Enter eventProcess!"; // qDebug()<< "Enter eventProcess!";
int avail; int avail;
if (ioctl(socket, FIONREAD, &avail) == EINVAL) { if (ioctl(socket, FIONREAD, &avail) == EINVAL) {
qWarning() << "Did not receive an entire inotify event."; qWarning() << "Did not receive an entire inotify event.";
@ -342,8 +345,8 @@ void InotifyWatch::eventProcess(int socket)
while (i < len) { while (i < len) {
const struct inotify_event* event = (struct inotify_event*)&buffer[i]; const struct inotify_event* event = (struct inotify_event*)&buffer[i];
if(event->name[0] != '.') { if(event->name[0] != '.') {
qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask; // qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
qDebug("mask:0x%x,",event->mask); // qDebug("mask:0x%x,",event->mask);
break; break;
} }
i += sizeof(struct inotify_event) + event->len; i += sizeof(struct inotify_event) + event->len;
@ -358,8 +361,7 @@ void InotifyWatch::eventProcess(int socket)
void InotifyWatch::eventProcess(const char *buffer, ssize_t len) void InotifyWatch::eventProcess(const char *buffer, ssize_t len)
{ {
qDebug()<< "Begin eventProcess! len:" << len; // qDebug()<< "Begin eventProcess! len:" << len;
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "0");
char * p = const_cast<char*>(buffer); char * p = const_cast<char*>(buffer);
while (p < buffer + len) { while (p < buffer + len) {
@ -370,7 +372,7 @@ void InotifyWatch::eventProcess(const char *buffer, ssize_t len)
QString path = currentPath[event->wd] + '/' + event->name; QString path = currentPath[event->wd] + '/' + event->name;
//Create top dir first, traverse it last. //Create top dir first, traverse it last.
if(event->mask & IN_CREATE) { if(event->mask & IN_CREATE) {
qDebug() << "IN_CREATE"; // qDebug() << "IN_CREATE";
PendingFile f(path); PendingFile f(path);
if(event->mask & IN_ISDIR) { if(event->mask & IN_ISDIR) {
f.setIsDir(); f.setIsDir();
@ -401,7 +403,7 @@ void InotifyWatch::eventProcess(const char *buffer, ssize_t len)
continue; continue;
} }
if(event->mask & IN_MODIFY) { if(event->mask & IN_MODIFY) {
qDebug() << "IN_MODIFY"; // qDebug() << "IN_MODIFY";
if(!(event->mask & IN_ISDIR)) { if(!(event->mask & IN_ISDIR)) {
PendingFileQueue::getInstance()->enqueue(PendingFile(path)); PendingFileQueue::getInstance()->enqueue(PendingFile(path));
} }
@ -437,7 +439,7 @@ void InotifyWatch::eventProcess(const char *buffer, ssize_t len)
next: next:
p += sizeof(struct inotify_event) + event->len; p += sizeof(struct inotify_event) + event->len;
} }
qDebug()<< "Finish eventProcess!"; // qDebug()<< "Finish eventProcess!";
} }

42
libsearch/index/pending-file-queue.cpp
View File

@ -40,10 +40,11 @@ PendingFileQueue::PendingFileQueue(QObject *parent) : QThread(parent)
// connect(this, &PendingFileQueue::minProcessTimerStart, m_minProcessTimer, f,Qt::DirectConnection); // connect(this, &PendingFileQueue::minProcessTimerStart, m_minProcessTimer, f,Qt::DirectConnection);
connect(this, SIGNAL(cacheTimerStart()), m_cacheTimer, SLOT(start())); connect(this, SIGNAL(cacheTimerStart()), m_cacheTimer, SLOT(start()));
connect(this, SIGNAL(minProcessTimerStart()), m_minProcessTimer, SLOT(start())); connect(this, SIGNAL(minProcessTimerStart()), m_minProcessTimer, SLOT(start()));
connect(this, &PendingFileQueue::timerStop, m_cacheTimer, &QTimer::stop);
connect(this, &PendingFileQueue::timerStop, m_minProcessTimer, &QTimer::stop);
connect(m_cacheTimer, &QTimer::timeout, this, &PendingFileQueue::processCache, Qt::DirectConnection); connect(m_cacheTimer, &QTimer::timeout, this, &PendingFileQueue::processCache, Qt::DirectConnection);
connect(m_minProcessTimer, &QTimer::timeout, this, &PendingFileQueue::processCache, Qt::DirectConnection); connect(m_minProcessTimer, &QTimer::timeout, this, &PendingFileQueue::processCache, Qt::DirectConnection);
} }
PendingFileQueue *PendingFileQueue::getInstance(QObject *parent) PendingFileQueue *PendingFileQueue::getInstance(QObject *parent)
@ -71,13 +72,18 @@ PendingFileQueue::~PendingFileQueue()
void PendingFileQueue::forceFinish() void PendingFileQueue::forceFinish()
{ {
QThread::msleep(600); QThread::msleep(600);
Q_EMIT timerStop();
this->quit(); this->quit();
this->wait(); this->wait();
} }
void PendingFileQueue::enqueue(const PendingFile &file) void PendingFileQueue::enqueue(const PendingFile &file)
{ {
qDebug() << "enqueuq file: " << file.path(); // qDebug() << "enqueuq file: " << file.path();
m_mutex.lock(); m_mutex.lock();
m_enqueuetimes++;
if(m_cache.isEmpty()) {
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "0");
}
// Remove all indexs of files under a dir which is to about be deleted,but keep delete signals. // Remove all indexs of files under a dir which is to about be deleted,but keep delete signals.
// Because our datebase need to delete those indexs one by one. // Because our datebase need to delete those indexs one by one.
if(file.shouldRemoveIndex() && file.isDir()) { if(file.shouldRemoveIndex() && file.isDir()) {
@ -94,29 +100,29 @@ void PendingFileQueue::enqueue(const PendingFile &file)
} }
int i = m_cache.indexOf(file); int i = m_cache.indexOf(file);
if (i == -1) { if (i == -1) {
qDebug() << "insert file" << file.path() << file.shouldRemoveIndex(); // qDebug() << "insert file" << file.path() << file.shouldRemoveIndex();
m_cache << file; m_cache << file;
} else { } else {
qDebug() << "merge file" << file.path() << file.shouldRemoveIndex(); // qDebug() << "merge file" << file.path() << file.shouldRemoveIndex();
m_cache[i].merge(file); m_cache[i].merge(file);
} }
if(!m_cacheTimer->isActive()) { if(!m_cacheTimer->isActive()) {
qDebug()<<"m_cacheTimer-----start!!"; // qDebug()<<"m_cacheTimer-----start!!";
// m_cacheTimer->start(); // m_cacheTimer->start();
Q_EMIT cacheTimerStart(); Q_EMIT cacheTimerStart();
} }
Q_EMIT minProcessTimerStart(); Q_EMIT minProcessTimerStart();
// m_minProcessTimer->start(); // m_minProcessTimer->start();
qDebug()<<"m_minProcessTimer-----start!!"; // qDebug()<<"m_minProcessTimer-----start!!";
m_mutex.unlock(); m_mutex.unlock();
qDebug() << "Current cache-------------"; // qDebug() << "Current cache-------------";
for(PendingFile i : m_cache) { // for(PendingFile i : m_cache) {
qDebug() << "|" << i.path(); // qDebug() << "|" << i.path();
qDebug() << "|" <<i.shouldRemoveIndex(); // qDebug() << "|" <<i.shouldRemoveIndex();
} // }
qDebug() << "Current cache-------------"; // qDebug() << "Current cache-------------";
qDebug()<<"enqueuq file finish!!"<<file.path(); // qDebug()<<"enqueuq file finish!!"<<file.path();
} }
void PendingFileQueue::run() void PendingFileQueue::run()
@ -126,13 +132,21 @@ void PendingFileQueue::run()
void PendingFileQueue::processCache() void PendingFileQueue::processCache()
{ {
qDebug()<< "Begin processCache!"; qDebug()<< "Begin processCache!" ;
m_mutex.lock(); m_mutex.lock();
qDebug() << "Events:" << m_enqueuetimes;
m_enqueuetimes = 0;
m_cache.swap(m_pendingFiles); m_cache.swap(m_pendingFiles);
// m_pendingFiles = m_cache; // m_pendingFiles = m_cache;
// m_cache.clear(); // m_cache.clear();
// m_cache.squeeze(); // m_cache.squeeze();
m_mutex.unlock(); m_mutex.unlock();
qDebug() << "Current process-------------";
for(PendingFile i : m_pendingFiles) {
qDebug() << "|" << i.path();
qDebug() << "|" <<i.shouldRemoveIndex();
}
qDebug() << "Current process-------------";
if(m_pendingFiles.isEmpty()) { if(m_pendingFiles.isEmpty()) {
qDebug()<< "Empty, finish processCache!"; qDebug()<< "Empty, finish processCache!";
return; return;

2
libsearch/index/pending-file-queue.h
View File

@ -48,6 +48,7 @@ protected:
Q_SIGNALS: Q_SIGNALS:
void cacheTimerStart(); void cacheTimerStart();
void minProcessTimerStart(); void minProcessTimerStart();
void timerStop();
private: private:
void processCache(); void processCache();
explicit PendingFileQueue(QObject *parent = nullptr); explicit PendingFileQueue(QObject *parent = nullptr);
@ -59,6 +60,7 @@ private:
QThread *m_timerThread = nullptr; QThread *m_timerThread = nullptr;
bool m_timeout = false; bool m_timeout = false;
int m_enqueuetimes = 0;
}; };
} }