openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

perf(file-index):增加一个额外的线程池用于实时更新索引

This commit is contained in:
iaom 2024-03-08 16:58:09 +08:00
parent 969c463418
commit 9efae17889
6 changed files with 33 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
libsearch/index/index-scheduler.cpp
View File

@ -38,6 +38,7 @@ IndexScheduler::IndexScheduler(QObject *parent) :
qRegisterMetaType<BatchIndexer::WorkMode>("WorkMode"); qRegisterMetaType<BatchIndexer::WorkMode>("WorkMode");
qRegisterMetaType<BatchIndexer::Targets>("Targets"); qRegisterMetaType<BatchIndexer::Targets>("Targets");
m_threadPool.setMaxThreadCount(1); m_threadPool.setMaxThreadCount(1);
m_threadPoolForRTUpdate.setMaxThreadCount(1);
connect(&m_fileWatcher, &FileWatcher::filesUpdate, this, &IndexScheduler::updateIndex); connect(&m_fileWatcher, &FileWatcher::filesUpdate, this, &IndexScheduler::updateIndex);
connect(m_config, &FileIndexerConfig::fileIndexEnableStatusChanged, this, &IndexScheduler::fileIndexEnable); connect(m_config, &FileIndexerConfig::fileIndexEnableStatusChanged, this, &IndexScheduler::fileIndexEnable);
connect(m_config, &FileIndexerConfig::contentIndexEnableStatusChanged, this, &IndexScheduler::contentIndexEnable); connect(m_config, &FileIndexerConfig::contentIndexEnableStatusChanged, this, &IndexScheduler::contentIndexEnable);
@ -122,6 +123,8 @@ void IndexScheduler::stop(BatchIndexer::Targets target)
m_fileWatcher.removeWatch(); m_fileWatcher.removeWatch();
m_threadPool.clear(); m_threadPool.clear();
m_threadPool.waitForDone(-1); m_threadPool.waitForDone(-1);
m_threadPoolForRTUpdate.clear();
m_threadPoolForRTUpdate.waitForDone(-1);
m_state = Stop; m_state = Stop;
qDebug() << "Index scheduler has been stopped."; qDebug() << "Index scheduler has been stopped.";
Q_EMIT stateChange(m_state); Q_EMIT stateChange(m_state);
@ -298,7 +301,7 @@ void IndexScheduler::updateIndex(const QVector<PendingFile> &files)
auto *updateJob = new IndexUpdater(files, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop); auto *updateJob = new IndexUpdater(files, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop);
connect(updateJob, &IndexUpdater::done, this, &IndexScheduler::updateFinished, Qt::QueuedConnection); connect(updateJob, &IndexUpdater::done, this, &IndexScheduler::updateFinished, Qt::QueuedConnection);
m_threadPool.start(updateJob); m_threadPoolForRTUpdate.start(updateJob);
} }
void IndexScheduler::batchIndexerFinished(BatchIndexer::WorkMode mode, BatchIndexer::Targets targets) void IndexScheduler::batchIndexerFinished(BatchIndexer::WorkMode mode, BatchIndexer::Targets targets)
@ -336,7 +339,8 @@ void IndexScheduler::onBasicIndexDone(BatchIndexer::WorkMode mode)
m_basicIndexPendingWorkCount--; m_basicIndexPendingWorkCount--;
bool success = false; bool success = false;
if(m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) { if(m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Initializing ||
m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Updating) {
m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
success = true; success = true;
} }
@ -350,7 +354,8 @@ void IndexScheduler::onContentIndexDone(BatchIndexer::WorkMode mode)
m_contentIndexPendingWorkCount--; m_contentIndexPendingWorkCount--;
bool success = false; bool success = false;
if(m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) { if(m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Initializing ||
m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Updating) {
m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
success = true; success = true;
} }
@ -364,7 +369,8 @@ void IndexScheduler::onOcrContentIndexDone(BatchIndexer::WorkMode mode)
m_ocrContentIndexPendingWorkCount--; m_ocrContentIndexPendingWorkCount--;
bool success = false; bool success = false;
if(m_statusRecorder->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) { if(m_statusRecorder->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Initializing ||
m_statusRecorder->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Updating) {
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
success = true; success = true;
} }

1
libsearch/index/index-scheduler.h
View File

@ -102,6 +102,7 @@ private:
QAtomicInt m_contentIndexStop; QAtomicInt m_contentIndexStop;
QAtomicInt m_ocrContentIndexStop; QAtomicInt m_ocrContentIndexStop;
QThreadPool m_threadPool; QThreadPool m_threadPool;
QThreadPool m_threadPoolForRTUpdate; //用于实时更新索引的线程池
//等待完成的工作数量 //等待完成的工作数量
quint64 m_basicIndexPendingWorkCount = 0; quint64 m_basicIndexPendingWorkCount = 0;

8
libsearch/index/index-status-recorder.cpp
View File

@ -48,8 +48,8 @@ const QVariant IndexStatusRecorder::getStatus(const QString &key)
{ {
m_mutex.lock(); m_mutex.lock();
m_status->sync(); m_status->sync();
m_mutex.unlock();
QVariant value = m_status->value(key); QVariant value = m_status->value(key);
m_mutex.unlock();
return value; return value;
} }
@ -74,8 +74,9 @@ bool IndexStatusRecorder::indexDatabaseEnable()
{ {
m_mutex.lock(); m_mutex.lock();
m_status->sync(); m_status->sync();
int state = m_status->value(INDEX_DATABASE_STATE_KEY, 0).toInt();
m_mutex.unlock(); m_mutex.unlock();
if(m_status->value(INDEX_DATABASE_STATE_KEY, 0).toInt() == State::Ready) { if(state == State::Ready) {
return true; return true;
} else { } else {
return false; return false;
@ -87,8 +88,9 @@ bool IndexStatusRecorder::contentIndexDatabaseEnable()
{ {
m_mutex.lock(); m_mutex.lock();
m_status->sync(); m_status->sync();
int state = m_status->value(CONTENT_INDEX_DATABASE_STATE_KEY, 0).toInt();
m_mutex.unlock(); m_mutex.unlock();
if(m_status->value(CONTENT_INDEX_DATABASE_STATE_KEY, 0).toInt() == State::Ready) { if(state == State::Ready) {
return true; return true;
} else { } else {
return false; return false;

3
libsearch/index/index-status-recorder.h
View File

@ -42,7 +42,8 @@ public:
Error = 1, Error = 1,
Ready = 2, Ready = 2,
Updating = 3, Updating = 3,
Off = 4 Off = 4,
RealTimeUpdating = 5
}; };
Q_ENUM(State) Q_ENUM(State)

14
libsearch/index/index-updater.cpp
View File

@ -26,6 +26,7 @@
#include "common.h" #include "common.h"
#include "file-utils.h" #include "file-utils.h"
#include "compatible-define.h" #include "compatible-define.h"
#include "index-status-recorder.h"
using namespace UkuiSearch; using namespace UkuiSearch;
IndexUpdater::IndexUpdater(const QVector<PendingFile>& files, QAtomicInt& indexstop, QAtomicInt& contentIndexstop, QAtomicInt& contentIndexOcrStop) IndexUpdater::IndexUpdater(const QVector<PendingFile>& files, QAtomicInt& indexstop, QAtomicInt& contentIndexstop, QAtomicInt& contentIndexOcrStop)
: m_cache(files), : m_cache(files),
@ -39,6 +40,7 @@ void IndexUpdater::updateIndex()
//fix me: How should I delete metadata of files below a folder //fix me: How should I delete metadata of files below a folder
//which has been deleted(When a file watcher signal comes which only contains folder info)? //which has been deleted(When a file watcher signal comes which only contains folder info)?
if(FileIndexerConfig::getInstance()->isFileIndexEnable() && !m_indexStop->LOAD) { if(FileIndexerConfig::getInstance()->isFileIndexEnable() && !m_indexStop->LOAD) {
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::RealTimeUpdating);
WritableDatabase basicDb(DataBaseType::Basic); WritableDatabase basicDb(DataBaseType::Basic);
if(!basicDb.open()) { if(!basicDb.open()) {
qWarning() << "Basic db open failed, fail to update index"; qWarning() << "Basic db open failed, fail to update index";
@ -62,6 +64,7 @@ void IndexUpdater::updateIndex()
} }
basicDb.commit(); basicDb.commit();
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
qDebug() << "===finish update basic index==="; qDebug() << "===finish update basic index===";
} }
} }
@ -69,8 +72,17 @@ void IndexUpdater::updateIndex()
void IndexUpdater::run() void IndexUpdater::run()
{ {
updateIndex(); updateIndex();
if(IndexStatusRecorder::getInstance()->getStatus(INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) {
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
}
updateContentIndex(); updateContentIndex();
if(IndexStatusRecorder::getInstance()->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) {
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
}
updateOcrContentIndex(); updateOcrContentIndex();
if(IndexStatusRecorder::getInstance()->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) {
IndexStatusRecorder::getInstance()->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
}
m_cache.clear(); m_cache.clear();
m_cache.shrink_to_fit(); m_cache.shrink_to_fit();
malloc_trim(0); malloc_trim(0);
@ -80,6 +92,7 @@ void IndexUpdater::run()
void IndexUpdater::updateContentIndex() void IndexUpdater::updateContentIndex()
{ {
if(FileIndexerConfig::getInstance()->isContentIndexEnable() && !m_contentIndexStop->LOAD) { if(FileIndexerConfig::getInstance()->isContentIndexEnable() && !m_contentIndexStop->LOAD) {
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::RealTimeUpdating);
WritableDatabase contentDb(DataBaseType::Content); WritableDatabase contentDb(DataBaseType::Content);
if(!contentDb.open()) { if(!contentDb.open()) {
qWarning() << "Content db open failed, fail to update index"; qWarning() << "Content db open failed, fail to update index";
@ -131,6 +144,7 @@ void IndexUpdater::updateContentIndex()
void IndexUpdater::updateOcrContentIndex() void IndexUpdater::updateOcrContentIndex()
{ {
if(FileIndexerConfig::getInstance()->isOCREnable() && !m_contentIndexOcrStop->LOAD) { if(FileIndexerConfig::getInstance()->isOCREnable() && !m_contentIndexOcrStop->LOAD) {
IndexStatusRecorder::getInstance()->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::RealTimeUpdating);
WritableDatabase contentDb(DataBaseType::OcrContent); WritableDatabase contentDb(DataBaseType::OcrContent);
if(!contentDb.open()) { if(!contentDb.open()) {
qWarning() << "Ocr content db open failed, fail to update index"; qWarning() << "Ocr content db open failed, fail to update index";

2
libsearch/searchinterface/searchtasks/file-content-search-task.cpp
View File

@ -81,7 +81,7 @@ void FileContentSearchTask::setEnable(bool enable)
bool FileContentSearchTask::isEnable() bool FileContentSearchTask::isEnable()
{ {
int state = IndexStatusRecorder::getInstance()->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt(); int state = IndexStatusRecorder::getInstance()->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt();
return m_enable && state == IndexStatusRecorder::State::Ready || state == IndexStatusRecorder::State::Updating; return m_enable && state == IndexStatusRecorder::State::Ready || state == IndexStatusRecorder::State::Updating || state == IndexStatusRecorder::State::RealTimeUpdating;
} }
QString FileContentSearchTask::getCustomSearchType() QString FileContentSearchTask::getCustomSearchType()