p90463782
/
ukui-search
forked from openkylin/ukui-search
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

修复反复开关索引导致的索引异常问题（偶现打开索引失败）

This commit is contained in:
iaom 2022-12-15 16:13:16 +08:00
parent 0ade94d032
commit 5eb29ea63d
8 changed files with 121 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
libsearch/index/first-run-indexer.cpp → libsearch/index/batch-indexer.cpp
View File

@ -17,7 +17,7 @@
* Authors: iaom <zhangpengfei@kylinos.cn>
*
*/
#include "first-run-indexer.h"
#include "batch-indexer.h"
#include <QFileInfo>
#include <QTime>
#include <malloc.h>
@ -27,8 +27,9 @@
#include "file-indexer-config.h"
#include "file-content-indexer.h"
#include "writable-database.h"
#include "compatible-define.h"
using namespace UkuiSearch;
FirstRunIndexer::FirstRunIndexer(const QStringList &folders, const QStringList &blackList, QAtomicInt& stop, WorkMode mode, Targets target)
BatchIndexer::BatchIndexer(const QStringList &folders, const QStringList &blackList, QAtomicInt& stop, WorkMode mode, Targets target)
: m_folders(folders),
m_blackList(blackList),
m_stop(&stop),
@ -37,14 +38,11 @@ FirstRunIndexer::FirstRunIndexer(const QStringList &folders, const QStringList &
{
}
FirstRunIndexer::~FirstRunIndexer()
{
}
void FirstRunIndexer::run()
void BatchIndexer::run()
{
QTime t = QTime::currentTime();
if(m_target == Target::None) {
if(m_target == Target::None || m_stop->LOAD) {
Q_EMIT done(m_mode);
return;
}
fetch();
@ -61,7 +59,7 @@ void FirstRunIndexer::run()
Q_EMIT done(m_mode);
}
void FirstRunIndexer::fetch()
void BatchIndexer::fetch()
{
qDebug() << "Now begin fetching files to be indexed...";
qDebug() << "Index folders:" << m_folders << "blacklist :" << m_blackList;
@ -107,7 +105,7 @@ void FirstRunIndexer::fetch()
qDebug() << m_cache.size() << "files founded, start index...";
}
void FirstRunIndexer::basicIndex()
void BatchIndexer::basicIndex()
{
qDebug() << "Begin basic index";
WritableDatabase basicDb(DataBaseType::Basic);
@ -161,6 +159,12 @@ void FirstRunIndexer::basicIndex()
qDebug() << "8192 finished.";
basicDb.commit();
Q_EMIT progress(IndexType::Basic, allSize, finishNum);
//文件名索引很快
if(m_stop->LOAD) {
qDebug() << "Index stopped, abort basic index.";
filesNeedIndex.clear();
return;
}
batchSize = 0;
}
}
@ -172,10 +176,10 @@ void FirstRunIndexer::basicIndex()
qDebug() << "Finish basic index";
}
void FirstRunIndexer::contentIndex()
void BatchIndexer::contentIndex()
{
qDebug() << "Begin content index";
if(m_stop->load()) {
if(m_stop->LOAD) {
qDebug() << "Index stopped, abort content index.";
return;
}
@ -242,6 +246,12 @@ void FirstRunIndexer::contentIndex()
uint batchSize = 0;
uint finishNum = 0;
for (QString path : filesNeedIndex) {
if(m_stop->LOAD) {
qDebug() << "Index stopped, interrupt content index.";
filesNeedIndex.clear();
filesNeedOCRIndex.clear();
return;
}
info.setFile(path);
if(true == targetPhotographTypeMap[info.suffix()]) {
filesNeedOCRIndex.append(path);
@ -262,10 +272,6 @@ void FirstRunIndexer::contentIndex()
Q_EMIT progress(IndexType::Contents, allSize, finishNum);
batchSize = 0;
}
if(m_stop->load()) {
qDebug() << "Index stopped, interrupt content index.";
break;
}
}
contentDb.commit();
Q_EMIT progress(IndexType::Contents, allSize, finishNum);
@ -278,6 +284,11 @@ void FirstRunIndexer::contentIndex()
batchSize = 0;
int ocrFinishNum = 0;
for(QString path : filesNeedOCRIndex) {
if(m_stop->LOAD) {
qDebug() << "Index stopped, interrupt content index.";
filesNeedOCRIndex.clear();
return;
}
fileContentIndexer indexer(path);
if(indexer.index()) {
contentDb.addDocument(indexer.document());
@ -293,10 +304,6 @@ void FirstRunIndexer::contentIndex()
Q_EMIT progress(IndexType::OCR, ocrSize, ocrFinishNum);
batchSize = 0;
}
if(m_stop->load()) {
qDebug() << "Index stopped, interrupt content index.";
break;
}
}
contentDb.commit();
Q_EMIT progress(IndexType::OCR, ocrSize, ocrFinishNum);

13
libsearch/index/first-run-indexer.h → libsearch/index/batch-indexer.h
View File

@ -26,7 +26,7 @@
#include "common.h"
namespace UkuiSearch {
class FirstRunIndexer : public QObject, public QRunnable
class BatchIndexer : public QObject, public QRunnable
{
Q_OBJECT
public:
@ -57,8 +57,7 @@ public:
};
Q_DECLARE_FLAGS(Targets, Target)
FirstRunIndexer(const QStringList& folders, const QStringList& blackList, QAtomicInt& stop, WorkMode mode = WorkMode::Update, Targets target = Target::All);
~FirstRunIndexer();
BatchIndexer(const QStringList& folders, const QStringList& blackList, QAtomicInt& stop, WorkMode mode = WorkMode::Update, Targets target = Target::All);
void run() override;
Q_SIGNALS:
@ -72,13 +71,13 @@ private:
void basicIndex();
void contentIndex();
WorkMode m_mode;
Targets m_target;
QStringList m_folders;
QStringList m_blackList;
QStringList m_cache;
QAtomicInt *m_stop = nullptr;
WorkMode m_mode;
Targets m_target;
QStringList m_cache;
};
Q_DECLARE_OPERATORS_FOR_FLAGS(FirstRunIndexer::Targets)
Q_DECLARE_OPERATORS_FOR_FLAGS(BatchIndexer::Targets)
}
#endif // FIRSTRUNINDEXER_H

12
libsearch/index/compatible-define.h Normal file
View File

@ -0,0 +1,12 @@
#ifndef COMPATIBLEDEFINE_H
#define COMPATIBLEDEFINE_H
#endif // COMPATIBLEDEFINE_H
#if (QT_VERSION < QT_VERSION_CHECK(5, 14, 0))
//T QAtomicInteger::load() const
#define LOAD load()
#endif
#if (QT_VERSION >= QT_VERSION_CHECK(5, 14, 0))
//T QAtomicInteger::loadRelaxed() const
#define LOAD loadRelaxed()
#endif

118
libsearch/index/index-scheduler.cpp
View File

@ -19,6 +19,7 @@
*/
#include "index-scheduler.h"
#include "index-updater.h"
#include "compatible-define.h"
using namespace UkuiSearch;
IndexScheduler::IndexScheduler(QObject *parent) :
@ -29,7 +30,7 @@ IndexScheduler::IndexScheduler(QObject *parent) :
m_stop(0)
{
qRegisterMetaType<IndexerState>("IndexerState");
qRegisterMetaType<FirstRunIndexer::WorkMode>("FirstRunIndexer::WorkMode");
qRegisterMetaType<BatchIndexer::WorkMode>("BatchIndexer::WorkMode");
m_threadPool.setMaxThreadCount(1);
connect(&m_fileWatcher, &FileWatcher::filesUpdate, this, &IndexScheduler::updateIndex);
connect(m_config, &FileIndexerConfig::fileIndexEnableStatusChanged, this, &IndexScheduler::fileIndexEnable);
@ -37,7 +38,7 @@ IndexScheduler::IndexScheduler(QObject *parent) :
connect(m_config, &FileIndexerConfig::removeIndexDir, this, &IndexScheduler::removeIndex);
m_state = Startup;
if(m_config->isFileIndexEnable()) {
scheduleIndexing();
start();
} else {
m_stop.fetchAndStoreRelaxed(1);
}
@ -45,30 +46,29 @@ IndexScheduler::IndexScheduler(QObject *parent) :
void IndexScheduler::addNewPath(const QString &folders, const QStringList &blackList)
{
if(m_stop.load()) {
if(m_stop.LOAD) {
qDebug() << "Index Scheduler is being stopped, add operation will be executed when started up next time.";
return;
}
m_isAddNewPathFinished = false;
m_state = Running;
Q_EMIT stateChange(m_state);
FirstRunIndexer::Targets target = FirstRunIndexer::Target::None;
BatchIndexer::Targets target = BatchIndexer::Target::None;
if(m_config->isFileIndexEnable()) {
target |= FirstRunIndexer::Target::Basic;
target |= BatchIndexer::Target::Basic;
}
if(m_config->isContentIndexEnable()) {
target |= FirstRunIndexer::Target::Content;
target |= BatchIndexer::Target::Content;
}
FirstRunIndexer::WorkMode mode = FirstRunIndexer::WorkMode::Add;
BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Add;
startIndexJob(QStringList() << folders, blackList, mode, target);
if(FirstRunIndexer::Target::None != target) {
if(BatchIndexer::Target::None != target) {
m_fileWatcher.addWatch(folders, blackList);
}
}
void IndexScheduler::removeIndex(const QString &folders)
{
if(m_stop.load()) {
if(m_stop.LOAD) {
qDebug() << "Index Scheduler is being stopped, remove operation will be executed when started up next time.";
return;
}
@ -80,33 +80,34 @@ void IndexScheduler::stop()
m_stop.fetchAndStoreRelaxed(1);
m_fileWatcher.removeWatch();
m_threadPool.clear();
m_threadPool.waitForDone(-1);
m_state = Stop;
qDebug() << "Index scheduler has been stopped.";
Q_EMIT stateChange(m_state);
}
void IndexScheduler::scheduleIndexing()
void IndexScheduler::start()
{
if(!m_isFirstRunFinished) {
qDebug() << "Index scheduler start.";
if(!m_isFirstRunFinished || !m_isRebuildFinished) {
qDebug() << "Index scheduler running, start operation ignored. FirstRun finished: " << m_isFirstRunFinished << "Rebuild finished: " << m_isRebuildFinished;
return;
}
m_isFirstRunFinished = false;
m_stop.fetchAndStoreRelaxed(0);
m_state = Running;
Q_EMIT stateChange(m_state);
FirstRunIndexer::Targets rebuiltTarget = checkAndRebuild();
BatchIndexer::Targets rebuiltTarget = checkAndRebuild();
FirstRunIndexer::WorkMode mode = FirstRunIndexer::WorkMode::Update;
FirstRunIndexer::Targets target = FirstRunIndexer::Target::None;
BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Update;
BatchIndexer::Targets target = BatchIndexer::Target::None;
//如果数据库被执行过重建，那么跳过增量更新步骤。
if(m_config->isFileIndexEnable() && !(rebuiltTarget & FirstRunIndexer::Target::Basic)) {
target |= FirstRunIndexer::Target::Basic;
if(m_config->isFileIndexEnable() && !(rebuiltTarget & BatchIndexer::Target::Basic)) {
target |= BatchIndexer::Target::Basic;
m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
}
if(m_config->isContentIndexEnable() && !(rebuiltTarget & FirstRunIndexer::Target::Content)) {
target |= FirstRunIndexer::Target::Content;
if(m_config->isContentIndexEnable() && !(rebuiltTarget & BatchIndexer::Target::Content)) {
target |= BatchIndexer::Target::Content;
m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
}
startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, target);
@ -120,35 +121,50 @@ IndexScheduler::IndexerState IndexScheduler::getIndexState()
return m_state;
}
FirstRunIndexer::Targets IndexScheduler::checkAndRebuild()
BatchIndexer::Targets IndexScheduler::checkAndRebuild()
{
FirstRunIndexer::WorkMode mode = FirstRunIndexer::WorkMode::Rebuild;
FirstRunIndexer::Targets target = FirstRunIndexer::Target::None;
if((m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error || !m_statusRecorder->versionCheck(INDEX_DATABASE_VERSION_KEY, INDEX_DATABASE_VERSION))
BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Rebuild;
BatchIndexer::Targets target = BatchIndexer::Target::None;
if((m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error
|| !m_statusRecorder->versionCheck(INDEX_DATABASE_VERSION_KEY, INDEX_DATABASE_VERSION))
&& m_config->isFileIndexEnable()) {
qDebug() << "Basic database need rebuild";
target |= FirstRunIndexer::Target::Basic;
target |= BatchIndexer::Target::Basic;
m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Initializing);
}
if((m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error || !m_statusRecorder->versionCheck(CONTENT_DATABASE_VERSION_KEY, CONTENT_DATABASE_VERSION))
if((m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error
|| !m_statusRecorder->versionCheck(CONTENT_DATABASE_VERSION_KEY, CONTENT_DATABASE_VERSION))
&& m_config->isFileIndexEnable()) {
qDebug() << "Content database need rebuild";
target |= FirstRunIndexer::Target::Content;
target |= BatchIndexer::Target::Content;
m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Initializing);
}
startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, target);
return target;
}
void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList& blackList, FirstRunIndexer::WorkMode mode, FirstRunIndexer::Targets target)
void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList& blackList, BatchIndexer::WorkMode mode, BatchIndexer::Targets target)
{
if(FirstRunIndexer::Target::None != target) {
FirstRunIndexer *indexer = new FirstRunIndexer(folders, blackList, m_stop, mode, target);
connect(indexer, &FirstRunIndexer::done, this, &IndexScheduler::firstRunFinished, Qt::QueuedConnection);
connect(indexer, &FirstRunIndexer::progress, this, &IndexScheduler::process);
if(BatchIndexer::Target::None != target) {
switch (mode) {
case BatchIndexer::WorkMode::Add:
m_isAddNewPathFinished = false;
break;
case BatchIndexer::WorkMode::Rebuild:
m_isRebuildFinished = false;
break;
case BatchIndexer::WorkMode::Update:
m_isFirstRunFinished = false;
break;
default:
break;
}
BatchIndexer *indexer = new BatchIndexer(folders, blackList, m_stop, mode, target);
connect(indexer, &BatchIndexer::done, this, &IndexScheduler::firstRunFinished, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::progress, this, &IndexScheduler::process);
connect(indexer, &FirstRunIndexer::basicIndexDone, this, [&](uint size){
connect(indexer, &BatchIndexer::basicIndexDone, this, [&](uint size){
bool success = false;
if(!(m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error)) {
m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
@ -157,7 +173,7 @@ void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList&
Q_EMIT basicIndexDone(size, success);
});
connect(indexer, &FirstRunIndexer::contentIndexDone, this, [&](uint size){
connect(indexer, &BatchIndexer::contentIndexDone, this, [&](uint size){
bool success = false;
if(!(m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error)) {
m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
@ -172,8 +188,9 @@ void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList&
void IndexScheduler::fileIndexEnable(bool enable)
{
//Fix me: 快速反复开关会导致反复执行增量更新操作，可优化。
if(enable) {
scheduleIndexing();
start();
} else {
stop();
}
@ -189,23 +206,20 @@ void IndexScheduler::updateIndex(const QVector<PendingFile> &files)
m_threadPool.start(updateJob);
}
void IndexScheduler::firstRunFinished(FirstRunIndexer::WorkMode mode)
void IndexScheduler::firstRunFinished(BatchIndexer::WorkMode mode)
{
if((m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Ready)
&& m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Ready) {
switch (mode) {
case FirstRunIndexer::WorkMode::Add:
m_isAddNewPathFinished = true;
break;
case FirstRunIndexer::WorkMode::Rebuild:
m_isRebuildFinished = true;
break;
case FirstRunIndexer::WorkMode::Update:
m_isFirstRunFinished = true;
break;
default:
break;
}
switch (mode) {
case BatchIndexer::WorkMode::Add:
m_isAddNewPathFinished = true;
break;
case BatchIndexer::WorkMode::Rebuild:
m_isRebuildFinished = true;
break;
case BatchIndexer::WorkMode::Update:
m_isFirstRunFinished = true;
break;
default:
break;
}
if(isIdle()) {
m_state = Idle;

10
libsearch/index/index-scheduler.h
View File

@ -26,7 +26,7 @@
#include "file-watcher.h"
#include "index-status-recorder.h"
#include "common.h"
#include "first-run-indexer.h"
#include "batch-indexer.h"
namespace UkuiSearch {
class IndexScheduler : public QObject
@ -55,7 +55,7 @@ public:
*/
Q_SCRIPTABLE void removeIndex(const QString& folders);
Q_SCRIPTABLE void stop();
Q_SCRIPTABLE void scheduleIndexing();
Q_SCRIPTABLE void start();
Q_SCRIPTABLE IndexerState getIndexState();
Q_SIGNALS:
@ -68,7 +68,7 @@ Q_SIGNALS:
private Q_SLOTS:
void fileIndexEnable(bool enable);
void updateIndex(const QVector<PendingFile>& files);
void firstRunFinished(FirstRunIndexer::WorkMode mode);
void firstRunFinished(BatchIndexer::WorkMode mode);
void updateFinished();
bool isIdle();
@ -78,8 +78,8 @@ private:
* 检查数据库状态，数据库状态处于 IndexStatusRecorder::State::Error 时，开始重建任务。
* @return 返回需要重建的数据库
*/
FirstRunIndexer::Targets checkAndRebuild();
void startIndexJob(const QStringList &folders, const QStringList &blackList, FirstRunIndexer::WorkMode mode, FirstRunIndexer::Targets target);
BatchIndexer::Targets checkAndRebuild();
void startIndexJob(const QStringList &folders, const QStringList &blackList, BatchIndexer::WorkMode mode, BatchIndexer::Targets target);
FileWatcher m_fileWatcher;
IndexStatusRecorder *m_statusRecorder = nullptr;
FileIndexerConfig *m_config = nullptr;

5
libsearch/index/index.pri
View File

@ -2,13 +2,14 @@ INCLUDEPATH += $$PWD
HEADERS += \
$$PWD/basic-indexer.h \
$$PWD/batch-indexer.h \
$$PWD/compatible-define.h \
$$PWD/database.h \
$$PWD/document.h \
# $$PWD/file-iterator.h \
$$PWD/file-content-indexer.h \
$$PWD/file-reader.h \
$$PWD/file-search-plugin.h \
$$PWD/first-run-indexer.h \
$$PWD/index-scheduler.h \
$$PWD/index-status-recorder.h \
$$PWD/monitor.h \
@ -24,12 +25,12 @@ HEADERS += \
SOURCES += \
$$PWD/basic-indexer.cpp \
$$PWD/batch-indexer.cpp \
$$PWD/database.cpp \
$$PWD/document.cpp \
$$PWD/file-content-indexer.cpp \
$$PWD/file-reader.cpp \
$$PWD/file-search-plugin.cpp \
$$PWD/first-run-indexer.cpp \
$$PWD/index-scheduler.cpp \
$$PWD/index-status-recorder.cpp \
$$PWD/monitor.cpp \

2
libsearch/libsearch.pro
View File

@ -88,7 +88,7 @@ preview.path = /usr/share/appwidget/search/
svg.files += search-app-widget-plugin/provider/data/ukui-search.svg
svg.path = /usr/share/appwidget/search/
INSTALLS += target qml qm_files appwidgetconf service preview svg
INSTALLS += qml qm_files appwidgetconf service preview svg
# Default rules for deployment.

2
ukui-search-service/ukui-search-service.cpp
View File

@ -83,7 +83,7 @@ void UkuiSearchService::parseCmd(QString msg, bool isPrimary)
if(parser.isSet(startOption)) {
qDebug() << "options!!!!" << parser.value(startOption);
if(parser.value(startOption) == "start") {
m_indexScheduler->scheduleIndexing();
m_indexScheduler->start();
} else if (parser.value(startOption) == "stop") {
m_indexScheduler->stop();
}