From 77395a862741494d2d5e399d4589eb3d67c35e55 Mon Sep 17 00:00:00 2001 From: iaom Date: Thu, 15 Dec 2022 16:13:16 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8F=8D=E5=A4=8D=E5=BC=80?= =?UTF-8?q?=E5=85=B3=E7=B4=A2=E5=BC=95=E5=AF=BC=E8=87=B4=E7=9A=84=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=E5=BC=82=E5=B8=B8=E9=97=AE=E9=A2=98=EF=BC=88=E5=81=B6?= =?UTF-8?q?=E7=8E=B0=E6=89=93=E5=BC=80=E7=B4=A2=E5=BC=95=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...irst-run-indexer.cpp => batch-indexer.cpp} | 47 ++++--- .../{first-run-indexer.h => batch-indexer.h} | 13 +- libsearch/index/compatible-define.h | 12 ++ libsearch/index/index-scheduler.cpp | 118 ++++++++++-------- libsearch/index/index-scheduler.h | 10 +- libsearch/index/index.pri | 5 +- libsearch/libsearch.pro | 2 +- ukui-search-service/ukui-search-service.cpp | 2 +- 8 files changed, 121 insertions(+), 88 deletions(-) rename libsearch/index/{first-run-indexer.cpp => batch-indexer.cpp} (92%) rename libsearch/index/{first-run-indexer.h => batch-indexer.h} (87%) create mode 100644 libsearch/index/compatible-define.h diff --git a/libsearch/index/first-run-indexer.cpp b/libsearch/index/batch-indexer.cpp similarity index 92% rename from libsearch/index/first-run-indexer.cpp rename to libsearch/index/batch-indexer.cpp index 282a7d0..2d74bcc 100644 --- a/libsearch/index/first-run-indexer.cpp +++ b/libsearch/index/batch-indexer.cpp @@ -17,7 +17,7 @@ * Authors: iaom * */ -#include "first-run-indexer.h" +#include "batch-indexer.h" #include #include #include @@ -27,8 +27,9 @@ #include "file-indexer-config.h" #include "file-content-indexer.h" #include "writable-database.h" +#include "compatible-define.h" using namespace UkuiSearch; -FirstRunIndexer::FirstRunIndexer(const QStringList &folders, const QStringList &blackList, QAtomicInt& stop, WorkMode mode, Targets target) +BatchIndexer::BatchIndexer(const QStringList &folders, const QStringList &blackList, QAtomicInt& stop, WorkMode mode, Targets target) : m_folders(folders), m_blackList(blackList), m_stop(&stop), @@ -37,14 +38,11 @@ FirstRunIndexer::FirstRunIndexer(const QStringList &folders, const QStringList & { } -FirstRunIndexer::~FirstRunIndexer() -{ -} - -void FirstRunIndexer::run() +void BatchIndexer::run() { QTime t = QTime::currentTime(); - if(m_target == Target::None) { + if(m_target == Target::None || m_stop->LOAD) { + Q_EMIT done(m_mode); return; } fetch(); @@ -61,7 +59,7 @@ void FirstRunIndexer::run() Q_EMIT done(m_mode); } -void FirstRunIndexer::fetch() +void BatchIndexer::fetch() { qDebug() << "Now begin fetching files to be indexed..."; qDebug() << "Index folders:" << m_folders << "blacklist :" << m_blackList; @@ -107,7 +105,7 @@ void FirstRunIndexer::fetch() qDebug() << m_cache.size() << "files founded, start index..."; } -void FirstRunIndexer::basicIndex() +void BatchIndexer::basicIndex() { qDebug() << "Begin basic index"; WritableDatabase basicDb(DataBaseType::Basic); @@ -161,6 +159,12 @@ void FirstRunIndexer::basicIndex() qDebug() << "8192 finished."; basicDb.commit(); Q_EMIT progress(IndexType::Basic, allSize, finishNum); + //文件名索引很快 + if(m_stop->LOAD) { + qDebug() << "Index stopped, abort basic index."; + filesNeedIndex.clear(); + return; + } batchSize = 0; } } @@ -172,10 +176,10 @@ void FirstRunIndexer::basicIndex() qDebug() << "Finish basic index"; } -void FirstRunIndexer::contentIndex() +void BatchIndexer::contentIndex() { qDebug() << "Begin content index"; - if(m_stop->load()) { + if(m_stop->LOAD) { qDebug() << "Index stopped, abort content index."; return; } @@ -242,6 +246,12 @@ void FirstRunIndexer::contentIndex() uint batchSize = 0; uint finishNum = 0; for (QString path : filesNeedIndex) { + if(m_stop->LOAD) { + qDebug() << "Index stopped, interrupt content index."; + filesNeedIndex.clear(); + filesNeedOCRIndex.clear(); + return; + } info.setFile(path); if(true == targetPhotographTypeMap[info.suffix()]) { filesNeedOCRIndex.append(path); @@ -262,10 +272,6 @@ void FirstRunIndexer::contentIndex() Q_EMIT progress(IndexType::Contents, allSize, finishNum); batchSize = 0; } - if(m_stop->load()) { - qDebug() << "Index stopped, interrupt content index."; - break; - } } contentDb.commit(); Q_EMIT progress(IndexType::Contents, allSize, finishNum); @@ -278,6 +284,11 @@ void FirstRunIndexer::contentIndex() batchSize = 0; int ocrFinishNum = 0; for(QString path : filesNeedOCRIndex) { + if(m_stop->LOAD) { + qDebug() << "Index stopped, interrupt content index."; + filesNeedOCRIndex.clear(); + return; + } fileContentIndexer indexer(path); if(indexer.index()) { contentDb.addDocument(indexer.document()); @@ -293,10 +304,6 @@ void FirstRunIndexer::contentIndex() Q_EMIT progress(IndexType::OCR, ocrSize, ocrFinishNum); batchSize = 0; } - if(m_stop->load()) { - qDebug() << "Index stopped, interrupt content index."; - break; - } } contentDb.commit(); Q_EMIT progress(IndexType::OCR, ocrSize, ocrFinishNum); diff --git a/libsearch/index/first-run-indexer.h b/libsearch/index/batch-indexer.h similarity index 87% rename from libsearch/index/first-run-indexer.h rename to libsearch/index/batch-indexer.h index c20c460..56361ab 100644 --- a/libsearch/index/first-run-indexer.h +++ b/libsearch/index/batch-indexer.h @@ -26,7 +26,7 @@ #include "common.h" namespace UkuiSearch { -class FirstRunIndexer : public QObject, public QRunnable +class BatchIndexer : public QObject, public QRunnable { Q_OBJECT public: @@ -57,8 +57,7 @@ public: }; Q_DECLARE_FLAGS(Targets, Target) - FirstRunIndexer(const QStringList& folders, const QStringList& blackList, QAtomicInt& stop, WorkMode mode = WorkMode::Update, Targets target = Target::All); - ~FirstRunIndexer(); + BatchIndexer(const QStringList& folders, const QStringList& blackList, QAtomicInt& stop, WorkMode mode = WorkMode::Update, Targets target = Target::All); void run() override; Q_SIGNALS: @@ -72,13 +71,13 @@ private: void basicIndex(); void contentIndex(); - WorkMode m_mode; - Targets m_target; QStringList m_folders; QStringList m_blackList; - QStringList m_cache; QAtomicInt *m_stop = nullptr; + WorkMode m_mode; + Targets m_target; + QStringList m_cache; }; -Q_DECLARE_OPERATORS_FOR_FLAGS(FirstRunIndexer::Targets) +Q_DECLARE_OPERATORS_FOR_FLAGS(BatchIndexer::Targets) } #endif // FIRSTRUNINDEXER_H diff --git a/libsearch/index/compatible-define.h b/libsearch/index/compatible-define.h new file mode 100644 index 0000000..bf23100 --- /dev/null +++ b/libsearch/index/compatible-define.h @@ -0,0 +1,12 @@ +#ifndef COMPATIBLEDEFINE_H +#define COMPATIBLEDEFINE_H + +#endif // COMPATIBLEDEFINE_H +#if (QT_VERSION < QT_VERSION_CHECK(5, 14, 0)) +//T QAtomicInteger::load() const +#define LOAD load() +#endif +#if (QT_VERSION >= QT_VERSION_CHECK(5, 14, 0)) +//T QAtomicInteger::loadRelaxed() const +#define LOAD loadRelaxed() +#endif diff --git a/libsearch/index/index-scheduler.cpp b/libsearch/index/index-scheduler.cpp index 6e74e91..6600abe 100644 --- a/libsearch/index/index-scheduler.cpp +++ b/libsearch/index/index-scheduler.cpp @@ -19,6 +19,7 @@ */ #include "index-scheduler.h" #include "index-updater.h" +#include "compatible-define.h" using namespace UkuiSearch; IndexScheduler::IndexScheduler(QObject *parent) : @@ -29,7 +30,7 @@ IndexScheduler::IndexScheduler(QObject *parent) : m_stop(0) { qRegisterMetaType("IndexerState"); - qRegisterMetaType("FirstRunIndexer::WorkMode"); + qRegisterMetaType("BatchIndexer::WorkMode"); m_threadPool.setMaxThreadCount(1); connect(&m_fileWatcher, &FileWatcher::filesUpdate, this, &IndexScheduler::updateIndex); connect(m_config, &FileIndexerConfig::fileIndexEnableStatusChanged, this, &IndexScheduler::fileIndexEnable); @@ -37,7 +38,7 @@ IndexScheduler::IndexScheduler(QObject *parent) : connect(m_config, &FileIndexerConfig::removeIndexDir, this, &IndexScheduler::removeIndex); m_state = Startup; if(m_config->isFileIndexEnable()) { - scheduleIndexing(); + start(); } else { m_stop.fetchAndStoreRelaxed(1); } @@ -45,30 +46,29 @@ IndexScheduler::IndexScheduler(QObject *parent) : void IndexScheduler::addNewPath(const QString &folders, const QStringList &blackList) { - if(m_stop.load()) { + if(m_stop.LOAD) { qDebug() << "Index Scheduler is being stopped, add operation will be executed when started up next time."; return; } - m_isAddNewPathFinished = false; m_state = Running; Q_EMIT stateChange(m_state); - FirstRunIndexer::Targets target = FirstRunIndexer::Target::None; + BatchIndexer::Targets target = BatchIndexer::Target::None; if(m_config->isFileIndexEnable()) { - target |= FirstRunIndexer::Target::Basic; + target |= BatchIndexer::Target::Basic; } if(m_config->isContentIndexEnable()) { - target |= FirstRunIndexer::Target::Content; + target |= BatchIndexer::Target::Content; } - FirstRunIndexer::WorkMode mode = FirstRunIndexer::WorkMode::Add; + BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Add; startIndexJob(QStringList() << folders, blackList, mode, target); - if(FirstRunIndexer::Target::None != target) { + if(BatchIndexer::Target::None != target) { m_fileWatcher.addWatch(folders, blackList); } } void IndexScheduler::removeIndex(const QString &folders) { - if(m_stop.load()) { + if(m_stop.LOAD) { qDebug() << "Index Scheduler is being stopped, remove operation will be executed when started up next time."; return; } @@ -80,33 +80,34 @@ void IndexScheduler::stop() m_stop.fetchAndStoreRelaxed(1); m_fileWatcher.removeWatch(); m_threadPool.clear(); + m_threadPool.waitForDone(-1); m_state = Stop; qDebug() << "Index scheduler has been stopped."; Q_EMIT stateChange(m_state); } -void IndexScheduler::scheduleIndexing() +void IndexScheduler::start() { - if(!m_isFirstRunFinished) { + qDebug() << "Index scheduler start."; + if(!m_isFirstRunFinished || !m_isRebuildFinished) { + qDebug() << "Index scheduler running, start operation ignored. FirstRun finished: " << m_isFirstRunFinished << "Rebuild finished: " << m_isRebuildFinished; return; } - - m_isFirstRunFinished = false; m_stop.fetchAndStoreRelaxed(0); m_state = Running; Q_EMIT stateChange(m_state); - FirstRunIndexer::Targets rebuiltTarget = checkAndRebuild(); + BatchIndexer::Targets rebuiltTarget = checkAndRebuild(); - FirstRunIndexer::WorkMode mode = FirstRunIndexer::WorkMode::Update; - FirstRunIndexer::Targets target = FirstRunIndexer::Target::None; + BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Update; + BatchIndexer::Targets target = BatchIndexer::Target::None; //如果数据库被执行过重建，那么跳过增量更新步骤。 - if(m_config->isFileIndexEnable() && !(rebuiltTarget & FirstRunIndexer::Target::Basic)) { - target |= FirstRunIndexer::Target::Basic; + if(m_config->isFileIndexEnable() && !(rebuiltTarget & BatchIndexer::Target::Basic)) { + target |= BatchIndexer::Target::Basic; m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating); } - if(m_config->isContentIndexEnable() && !(rebuiltTarget & FirstRunIndexer::Target::Content)) { - target |= FirstRunIndexer::Target::Content; + if(m_config->isContentIndexEnable() && !(rebuiltTarget & BatchIndexer::Target::Content)) { + target |= BatchIndexer::Target::Content; m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating); } startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, target); @@ -120,35 +121,50 @@ IndexScheduler::IndexerState IndexScheduler::getIndexState() return m_state; } -FirstRunIndexer::Targets IndexScheduler::checkAndRebuild() +BatchIndexer::Targets IndexScheduler::checkAndRebuild() { - FirstRunIndexer::WorkMode mode = FirstRunIndexer::WorkMode::Rebuild; - FirstRunIndexer::Targets target = FirstRunIndexer::Target::None; - if((m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error || !m_statusRecorder->versionCheck(INDEX_DATABASE_VERSION_KEY, INDEX_DATABASE_VERSION)) + BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Rebuild; + BatchIndexer::Targets target = BatchIndexer::Target::None; + if((m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error + || !m_statusRecorder->versionCheck(INDEX_DATABASE_VERSION_KEY, INDEX_DATABASE_VERSION)) && m_config->isFileIndexEnable()) { qDebug() << "Basic database need rebuild"; - target |= FirstRunIndexer::Target::Basic; + target |= BatchIndexer::Target::Basic; m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Initializing); } - if((m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error || !m_statusRecorder->versionCheck(CONTENT_DATABASE_VERSION_KEY, CONTENT_DATABASE_VERSION)) + if((m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error + || !m_statusRecorder->versionCheck(CONTENT_DATABASE_VERSION_KEY, CONTENT_DATABASE_VERSION)) && m_config->isFileIndexEnable()) { qDebug() << "Content database need rebuild"; - target |= FirstRunIndexer::Target::Content; + target |= BatchIndexer::Target::Content; m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Initializing); } startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, target); return target; } -void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList& blackList, FirstRunIndexer::WorkMode mode, FirstRunIndexer::Targets target) +void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList& blackList, BatchIndexer::WorkMode mode, BatchIndexer::Targets target) { - if(FirstRunIndexer::Target::None != target) { - FirstRunIndexer *indexer = new FirstRunIndexer(folders, blackList, m_stop, mode, target); - connect(indexer, &FirstRunIndexer::done, this, &IndexScheduler::firstRunFinished, Qt::QueuedConnection); - connect(indexer, &FirstRunIndexer::progress, this, &IndexScheduler::process); + if(BatchIndexer::Target::None != target) { + switch (mode) { + case BatchIndexer::WorkMode::Add: + m_isAddNewPathFinished = false; + break; + case BatchIndexer::WorkMode::Rebuild: + m_isRebuildFinished = false; + break; + case BatchIndexer::WorkMode::Update: + m_isFirstRunFinished = false; + break; + default: + break; + } + BatchIndexer *indexer = new BatchIndexer(folders, blackList, m_stop, mode, target); + connect(indexer, &BatchIndexer::done, this, &IndexScheduler::firstRunFinished, Qt::QueuedConnection); + connect(indexer, &BatchIndexer::progress, this, &IndexScheduler::process); - connect(indexer, &FirstRunIndexer::basicIndexDone, this, [&](uint size){ + connect(indexer, &BatchIndexer::basicIndexDone, this, [&](uint size){ bool success = false; if(!(m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error)) { m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); @@ -157,7 +173,7 @@ void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList& Q_EMIT basicIndexDone(size, success); }); - connect(indexer, &FirstRunIndexer::contentIndexDone, this, [&](uint size){ + connect(indexer, &BatchIndexer::contentIndexDone, this, [&](uint size){ bool success = false; if(!(m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error)) { m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); @@ -172,8 +188,9 @@ void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList& void IndexScheduler::fileIndexEnable(bool enable) { + //Fix me: 快速反复开关会导致反复执行增量更新操作，可优化。 if(enable) { - scheduleIndexing(); + start(); } else { stop(); } @@ -189,23 +206,20 @@ void IndexScheduler::updateIndex(const QVector &files) m_threadPool.start(updateJob); } -void IndexScheduler::firstRunFinished(FirstRunIndexer::WorkMode mode) +void IndexScheduler::firstRunFinished(BatchIndexer::WorkMode mode) { - if((m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Ready) - && m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Ready) { - switch (mode) { - case FirstRunIndexer::WorkMode::Add: - m_isAddNewPathFinished = true; - break; - case FirstRunIndexer::WorkMode::Rebuild: - m_isRebuildFinished = true; - break; - case FirstRunIndexer::WorkMode::Update: - m_isFirstRunFinished = true; - break; - default: - break; - } + switch (mode) { + case BatchIndexer::WorkMode::Add: + m_isAddNewPathFinished = true; + break; + case BatchIndexer::WorkMode::Rebuild: + m_isRebuildFinished = true; + break; + case BatchIndexer::WorkMode::Update: + m_isFirstRunFinished = true; + break; + default: + break; } if(isIdle()) { m_state = Idle; diff --git a/libsearch/index/index-scheduler.h b/libsearch/index/index-scheduler.h index 960783b..1a074a7 100644 --- a/libsearch/index/index-scheduler.h +++ b/libsearch/index/index-scheduler.h @@ -26,7 +26,7 @@ #include "file-watcher.h" #include "index-status-recorder.h" #include "common.h" -#include "first-run-indexer.h" +#include "batch-indexer.h" namespace UkuiSearch { class IndexScheduler : public QObject @@ -55,7 +55,7 @@ public: */ Q_SCRIPTABLE void removeIndex(const QString& folders); Q_SCRIPTABLE void stop(); - Q_SCRIPTABLE void scheduleIndexing(); + Q_SCRIPTABLE void start(); Q_SCRIPTABLE IndexerState getIndexState(); Q_SIGNALS: @@ -68,7 +68,7 @@ Q_SIGNALS: private Q_SLOTS: void fileIndexEnable(bool enable); void updateIndex(const QVector& files); - void firstRunFinished(FirstRunIndexer::WorkMode mode); + void firstRunFinished(BatchIndexer::WorkMode mode); void updateFinished(); bool isIdle(); @@ -78,8 +78,8 @@ private: * 检查数据库状态，数据库状态处于 IndexStatusRecorder::State::Error 时，开始重建任务。 * @return 返回需要重建的数据库 */ - FirstRunIndexer::Targets checkAndRebuild(); - void startIndexJob(const QStringList &folders, const QStringList &blackList, FirstRunIndexer::WorkMode mode, FirstRunIndexer::Targets target); + BatchIndexer::Targets checkAndRebuild(); + void startIndexJob(const QStringList &folders, const QStringList &blackList, BatchIndexer::WorkMode mode, BatchIndexer::Targets target); FileWatcher m_fileWatcher; IndexStatusRecorder *m_statusRecorder = nullptr; FileIndexerConfig *m_config = nullptr; diff --git a/libsearch/index/index.pri b/libsearch/index/index.pri index cdd3b31..601c063 100644 --- a/libsearch/index/index.pri +++ b/libsearch/index/index.pri @@ -2,13 +2,14 @@ INCLUDEPATH += $$PWD HEADERS += \ $$PWD/basic-indexer.h \ + $$PWD/batch-indexer.h \ + $$PWD/compatible-define.h \ $$PWD/database.h \ $$PWD/document.h \ # $$PWD/file-iterator.h \ $$PWD/file-content-indexer.h \ $$PWD/file-reader.h \ $$PWD/file-search-plugin.h \ - $$PWD/first-run-indexer.h \ $$PWD/index-scheduler.h \ $$PWD/index-status-recorder.h \ $$PWD/monitor.h \ @@ -24,12 +25,12 @@ HEADERS += \ SOURCES += \ $$PWD/basic-indexer.cpp \ + $$PWD/batch-indexer.cpp \ $$PWD/database.cpp \ $$PWD/document.cpp \ $$PWD/file-content-indexer.cpp \ $$PWD/file-reader.cpp \ $$PWD/file-search-plugin.cpp \ - $$PWD/first-run-indexer.cpp \ $$PWD/index-scheduler.cpp \ $$PWD/index-status-recorder.cpp \ $$PWD/monitor.cpp \ diff --git a/libsearch/libsearch.pro b/libsearch/libsearch.pro index d9fdb8e..f8e5466 100644 --- a/libsearch/libsearch.pro +++ b/libsearch/libsearch.pro @@ -88,7 +88,7 @@ preview.path = /usr/share/appwidget/search/ svg.files += search-app-widget-plugin/provider/data/ukui-search.svg svg.path = /usr/share/appwidget/search/ -INSTALLS += target qml qm_files appwidgetconf service preview svg +INSTALLS += qml qm_files appwidgetconf service preview svg # Default rules for deployment. diff --git a/ukui-search-service/ukui-search-service.cpp b/ukui-search-service/ukui-search-service.cpp index 54d0dab..9eb6985 100644 --- a/ukui-search-service/ukui-search-service.cpp +++ b/ukui-search-service/ukui-search-service.cpp @@ -83,7 +83,7 @@ void UkuiSearchService::parseCmd(QString msg, bool isPrimary) if(parser.isSet(startOption)) { qDebug() << "options!!!!" << parser.value(startOption); if(parser.value(startOption) == "start") { - m_indexScheduler->scheduleIndexing(); + m_indexScheduler->start(); } else if (parser.value(startOption) == "stop") { m_indexScheduler->stop(); }