From a5fce404d655c5de0eedb8a2f5ddb23c72e21403 Mon Sep 17 00:00:00 2001 From: iaom Date: Fri, 8 Mar 2024 16:58:09 +0800 Subject: [PATCH] =?UTF-8?q?perf(file-index):=E5=A2=9E=E5=8A=A0=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E9=A2=9D=E5=A4=96=E7=9A=84=E7=BA=BF=E7=A8=8B=E6=B1=A0?= =?UTF-8?q?=E7=94=A8=E4=BA=8E=E5=AE=9E=E6=97=B6=E6=9B=B4=E6=96=B0=E7=B4=A2?= =?UTF-8?q?=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libsearch/index/index-scheduler.cpp | 14 ++++++++++---- libsearch/index/index-scheduler.h | 1 + libsearch/index/index-status-recorder.cpp | 8 +++++--- libsearch/index/index-status-recorder.h | 3 ++- libsearch/index/index-updater.cpp | 14 ++++++++++++++ .../searchtasks/file-content-search-task.cpp | 2 +- 6 files changed, 33 insertions(+), 9 deletions(-) diff --git a/libsearch/index/index-scheduler.cpp b/libsearch/index/index-scheduler.cpp index c722331..8f47fca 100644 --- a/libsearch/index/index-scheduler.cpp +++ b/libsearch/index/index-scheduler.cpp @@ -38,6 +38,7 @@ IndexScheduler::IndexScheduler(QObject *parent) : qRegisterMetaType("WorkMode"); qRegisterMetaType("Targets"); m_threadPool.setMaxThreadCount(1); + m_threadPoolForRTUpdate.setMaxThreadCount(1); connect(&m_fileWatcher, &FileWatcher::filesUpdate, this, &IndexScheduler::updateIndex); connect(m_config, &FileIndexerConfig::fileIndexEnableStatusChanged, this, &IndexScheduler::fileIndexEnable); connect(m_config, &FileIndexerConfig::contentIndexEnableStatusChanged, this, &IndexScheduler::contentIndexEnable); @@ -122,6 +123,8 @@ void IndexScheduler::stop(BatchIndexer::Targets target) m_fileWatcher.removeWatch(); m_threadPool.clear(); m_threadPool.waitForDone(-1); + m_threadPoolForRTUpdate.clear(); + m_threadPoolForRTUpdate.waitForDone(-1); m_state = Stop; qDebug() << "Index scheduler has been stopped."; Q_EMIT stateChange(m_state); @@ -298,7 +301,7 @@ void IndexScheduler::updateIndex(const QVector &files) auto *updateJob = new IndexUpdater(files, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop); connect(updateJob, &IndexUpdater::done, this, &IndexScheduler::updateFinished, Qt::QueuedConnection); - m_threadPool.start(updateJob); + m_threadPoolForRTUpdate.start(updateJob); } void IndexScheduler::batchIndexerFinished(BatchIndexer::WorkMode mode, BatchIndexer::Targets targets) @@ -336,7 +339,8 @@ void IndexScheduler::onBasicIndexDone(BatchIndexer::WorkMode mode) m_basicIndexPendingWorkCount--; bool success = false; - if(m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) { + if(m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Initializing || + m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Updating) { m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); success = true; } @@ -350,7 +354,8 @@ void IndexScheduler::onContentIndexDone(BatchIndexer::WorkMode mode) m_contentIndexPendingWorkCount--; bool success = false; - if(m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) { + if(m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Initializing || + m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Updating) { m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); success = true; } @@ -364,7 +369,8 @@ void IndexScheduler::onOcrContentIndexDone(BatchIndexer::WorkMode mode) m_ocrContentIndexPendingWorkCount--; bool success = false; - if(m_statusRecorder->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) { + if(m_statusRecorder->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Initializing || + m_statusRecorder->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Updating) { m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); success = true; } diff --git a/libsearch/index/index-scheduler.h b/libsearch/index/index-scheduler.h index 7c3c82a..41eb616 100644 --- a/libsearch/index/index-scheduler.h +++ b/libsearch/index/index-scheduler.h @@ -102,6 +102,7 @@ private: QAtomicInt m_contentIndexStop; QAtomicInt m_ocrContentIndexStop; QThreadPool m_threadPool; + QThreadPool m_threadPoolForRTUpdate; //用于实时更新索引的线程池 //等待完成的工作数量 quint64 m_basicIndexPendingWorkCount = 0; diff --git a/libsearch/index/index-status-recorder.cpp b/libsearch/index/index-status-recorder.cpp index 77d012f..a420942 100644 --- a/libsearch/index/index-status-recorder.cpp +++ b/libsearch/index/index-status-recorder.cpp @@ -48,8 +48,8 @@ const QVariant IndexStatusRecorder::getStatus(const QString &key) { m_mutex.lock(); m_status->sync(); - m_mutex.unlock(); QVariant value = m_status->value(key); + m_mutex.unlock(); return value; } @@ -74,8 +74,9 @@ bool IndexStatusRecorder::indexDatabaseEnable() { m_mutex.lock(); m_status->sync(); + int state = m_status->value(INDEX_DATABASE_STATE_KEY, 0).toInt(); m_mutex.unlock(); - if(m_status->value(INDEX_DATABASE_STATE_KEY, 0).toInt() == State::Ready) { + if(state == State::Ready) { return true; } else { return false; @@ -87,8 +88,9 @@ bool IndexStatusRecorder::contentIndexDatabaseEnable() { m_mutex.lock(); m_status->sync(); + int state = m_status->value(CONTENT_INDEX_DATABASE_STATE_KEY, 0).toInt(); m_mutex.unlock(); - if(m_status->value(CONTENT_INDEX_DATABASE_STATE_KEY, 0).toInt() == State::Ready) { + if(state == State::Ready) { return true; } else { return false; diff --git a/libsearch/index/index-status-recorder.h b/libsearch/index/index-status-recorder.h index 1aa96a1..77d039e 100644 --- a/libsearch/index/index-status-recorder.h +++ b/libsearch/index/index-status-recorder.h @@ -42,7 +42,8 @@ public: Error = 1, Ready = 2, Updating = 3, - Off = 4 + Off = 4, + RealTimeUpdating = 5 }; Q_ENUM(State) diff --git a/libsearch/index/index-updater.cpp b/libsearch/index/index-updater.cpp index c2a3bce..53a3e92 100644 --- a/libsearch/index/index-updater.cpp +++ b/libsearch/index/index-updater.cpp @@ -26,6 +26,7 @@ #include "common.h" #include "file-utils.h" #include "compatible-define.h" +#include "index-status-recorder.h" using namespace UkuiSearch; IndexUpdater::IndexUpdater(const QVector& files, QAtomicInt& indexstop, QAtomicInt& contentIndexstop, QAtomicInt& contentIndexOcrStop) : m_cache(files), @@ -39,6 +40,7 @@ void IndexUpdater::updateIndex() //fix me: How should I delete metadata of files below a folder //which has been deleted(When a file watcher signal comes which only contains folder info)? if(FileIndexerConfig::getInstance()->isFileIndexEnable() && !m_indexStop->LOAD) { + IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::RealTimeUpdating); WritableDatabase basicDb(DataBaseType::Basic); if(!basicDb.open()) { qWarning() << "Basic db open failed, fail to update index"; @@ -62,6 +64,7 @@ void IndexUpdater::updateIndex() } basicDb.commit(); + IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); qDebug() << "===finish update basic index==="; } } @@ -69,8 +72,17 @@ void IndexUpdater::updateIndex() void IndexUpdater::run() { updateIndex(); + if(IndexStatusRecorder::getInstance()->getStatus(INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) { + IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); + } updateContentIndex(); + if(IndexStatusRecorder::getInstance()->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) { + IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); + } updateOcrContentIndex(); + if(IndexStatusRecorder::getInstance()->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) { + IndexStatusRecorder::getInstance()->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); + } m_cache.clear(); m_cache.shrink_to_fit(); malloc_trim(0); @@ -80,6 +92,7 @@ void IndexUpdater::run() void IndexUpdater::updateContentIndex() { if(FileIndexerConfig::getInstance()->isContentIndexEnable() && !m_contentIndexStop->LOAD) { + IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::RealTimeUpdating); WritableDatabase contentDb(DataBaseType::Content); if(!contentDb.open()) { qWarning() << "Content db open failed, fail to update index"; @@ -131,6 +144,7 @@ void IndexUpdater::updateContentIndex() void IndexUpdater::updateOcrContentIndex() { if(FileIndexerConfig::getInstance()->isOCREnable() && !m_contentIndexOcrStop->LOAD) { + IndexStatusRecorder::getInstance()->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::RealTimeUpdating); WritableDatabase contentDb(DataBaseType::OcrContent); if(!contentDb.open()) { qWarning() << "Ocr content db open failed, fail to update index"; diff --git a/libsearch/searchinterface/searchtasks/file-content-search-task.cpp b/libsearch/searchinterface/searchtasks/file-content-search-task.cpp index 9ca086d..a77dd6d 100644 --- a/libsearch/searchinterface/searchtasks/file-content-search-task.cpp +++ b/libsearch/searchinterface/searchtasks/file-content-search-task.cpp @@ -81,7 +81,7 @@ void FileContentSearchTask::setEnable(bool enable) bool FileContentSearchTask::isEnable() { int state = IndexStatusRecorder::getInstance()->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt(); - return m_enable && state == IndexStatusRecorder::State::Ready || state == IndexStatusRecorder::State::Updating; + return m_enable && state == IndexStatusRecorder::State::Ready || state == IndexStatusRecorder::State::Updating || state == IndexStatusRecorder::State::RealTimeUpdating; } QString FileContentSearchTask::getCustomSearchType()