404 lines
15 KiB
C++
404 lines
15 KiB
C++
/*
|
|
* Copyright (C) 2022, KylinSoft Co., Ltd.
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*
|
|
* Authors: iaom <zhangpengfei@kylinos.cn>
|
|
*
|
|
*/
|
|
#include "index-scheduler.h"
|
|
#include <QCoreApplication>
|
|
#include <QMetaEnum>
|
|
#include "index-updater.h"
|
|
#include "compatible-define.h"
|
|
|
|
using namespace UkuiSearch;
|
|
IndexScheduler::IndexScheduler(QObject *parent) :
|
|
QObject(parent),
|
|
m_statusRecorder(IndexStatusRecorder::getInstance()),
|
|
m_config(FileIndexerConfig::getInstance()),
|
|
m_state(Startup),
|
|
m_indexStop(0),
|
|
m_contentIndexStop(0),
|
|
m_ocrContentIndexStop(0)
|
|
{
|
|
qRegisterMetaType<IndexerState>("IndexerState");
|
|
qRegisterMetaType<BatchIndexer::WorkMode>("BatchIndexer::WorkMode");
|
|
qRegisterMetaType<BatchIndexer::WorkMode>("WorkMode");
|
|
qRegisterMetaType<BatchIndexer::Targets>("Targets");
|
|
m_threadPool.setMaxThreadCount(1);
|
|
connect(&m_fileWatcher, &FileWatcher::filesUpdate, this, &IndexScheduler::updateIndex);
|
|
connect(m_config, &FileIndexerConfig::fileIndexEnableStatusChanged, this, &IndexScheduler::fileIndexEnable);
|
|
connect(m_config, &FileIndexerConfig::contentIndexEnableStatusChanged, this, &IndexScheduler::contentIndexEnable);
|
|
connect(m_config, &FileIndexerConfig::contentIndexEnableOcrStatusChanged, this, &IndexScheduler::ocrContentIndexEnable);
|
|
|
|
connect(m_config, &FileIndexerConfig::appendIndexDir, this, &IndexScheduler::addNewPath);
|
|
connect(m_config, &FileIndexerConfig::removeIndexDir, this, &IndexScheduler::removeIndex);
|
|
m_state = Startup;
|
|
BatchIndexer::Targets targets = BatchIndexer::Target::None;
|
|
if(m_config->isFileIndexEnable()) {
|
|
targets |= BatchIndexer::Target::Basic;
|
|
} else {
|
|
m_indexStop.fetchAndStoreRelaxed(1);
|
|
}
|
|
if(m_config->isContentIndexEnable()) {
|
|
targets |= BatchIndexer::Target::Content;
|
|
} else {
|
|
m_contentIndexStop.fetchAndStoreRelaxed(1);
|
|
}
|
|
if(m_config->isOCREnable()) {
|
|
targets |= BatchIndexer::Target::Ocr;
|
|
} else {
|
|
m_ocrContentIndexStop.fetchAndStoreRelaxed(1);
|
|
}
|
|
start(targets);
|
|
}
|
|
|
|
void IndexScheduler::addNewPath(const QString &folders, const QStringList &blackList)
|
|
{
|
|
if(m_indexStop.LOAD && m_contentIndexStop.LOAD && m_ocrContentIndexStop.LOAD) {
|
|
qDebug() << "Index Scheduler is being stopped, add operation will be executed when started up next time.";
|
|
return;
|
|
}
|
|
|
|
BatchIndexer::Targets target = BatchIndexer::Target::None;
|
|
if(m_config->isFileIndexEnable()) {
|
|
target |= BatchIndexer::Target::Basic;
|
|
m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
|
|
}
|
|
if(m_config->isContentIndexEnable()) {
|
|
target |= BatchIndexer::Target::Content;
|
|
m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
|
|
}
|
|
if(m_config->isOCREnable()) {
|
|
target |= BatchIndexer::Target::Ocr;
|
|
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
|
|
}
|
|
BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Add;
|
|
startIndexJob(QStringList() << folders, blackList, mode, target);
|
|
if(BatchIndexer::Target::None != target) {
|
|
m_fileWatcher.addWatch(folders, blackList);
|
|
}
|
|
}
|
|
|
|
void IndexScheduler::removeIndex(const QString &folders)
|
|
{
|
|
if(m_indexStop.LOAD && m_contentIndexStop.LOAD && m_ocrContentIndexStop.LOAD) {
|
|
qDebug() << "Index Scheduler is being stopped, remove operation will be executed when started up next time.";
|
|
return;
|
|
}
|
|
m_fileWatcher.removeWatch(folders, true);
|
|
}
|
|
|
|
void IndexScheduler::stop(BatchIndexer::Targets target)
|
|
{
|
|
if(target & BatchIndexer::Target::Basic) {
|
|
m_indexStop.fetchAndStoreRelaxed(1);
|
|
m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Off);
|
|
qDebug() << "File index has been stopped.";
|
|
}
|
|
if(target & BatchIndexer::Target::Content) {
|
|
m_contentIndexStop.fetchAndStoreRelaxed(1);
|
|
m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Off);
|
|
qDebug() << "File content index has been stopped.";
|
|
}
|
|
if(target & BatchIndexer::Target::Ocr) {
|
|
m_ocrContentIndexStop.fetchAndStoreRelaxed(1);
|
|
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Off);
|
|
qDebug() << "File ocr content index has been stopped.";
|
|
}
|
|
if(m_indexStop.LOAD && m_contentIndexStop.LOAD && m_ocrContentIndexStop.LOAD) {
|
|
m_fileWatcher.removeWatch();
|
|
m_threadPool.clear();
|
|
m_threadPool.waitForDone(-1);
|
|
m_state = Stop;
|
|
qDebug() << "Index scheduler has been stopped.";
|
|
Q_EMIT stateChange(m_state);
|
|
}
|
|
}
|
|
|
|
IndexScheduler::IndexerState IndexScheduler::getIndexState()
|
|
{
|
|
return m_state;
|
|
}
|
|
|
|
void IndexScheduler::forceUpdate(BatchIndexer::Targets target)
|
|
{
|
|
m_config->sync();
|
|
stop(target);
|
|
if(target & BatchIndexer::Basic) {
|
|
fileIndexEnable(true);
|
|
}
|
|
if(target & BatchIndexer::Content) {
|
|
contentIndexEnable(true);
|
|
}
|
|
if(target & BatchIndexer::Ocr) {
|
|
ocrContentIndexEnable(true);
|
|
}
|
|
}
|
|
|
|
void IndexScheduler::start(BatchIndexer::Targets target)
|
|
{
|
|
qDebug() << "Index scheduler start." << QMetaEnum::fromType<BatchIndexer::Targets>().valueToKeys(target) ;
|
|
|
|
//打开异步控制开关
|
|
if(target & BatchIndexer::Basic) {
|
|
m_indexStop.fetchAndStoreRelaxed(0);
|
|
}
|
|
if(target & BatchIndexer::Content) {
|
|
m_contentIndexStop.fetchAndStoreRelaxed(0);
|
|
}
|
|
if(target & BatchIndexer::Ocr) {
|
|
m_ocrContentIndexStop.fetchAndStoreRelaxed(0);
|
|
}
|
|
if(target == BatchIndexer::None) {
|
|
return;
|
|
}
|
|
|
|
//检查是否有数据库需要重建并且执行重建
|
|
BatchIndexer::Targets rebuiltTarget = checkAndRebuild(target);
|
|
|
|
BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Update;
|
|
BatchIndexer::Targets startTarget = BatchIndexer::Target::None;
|
|
|
|
//如果数据库被执行过重建，那么跳过增量更新步骤。
|
|
if((target & BatchIndexer::Target::Basic) && !(rebuiltTarget & BatchIndexer::Target::Basic)) {
|
|
startTarget |= BatchIndexer::Target::Basic;
|
|
m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
|
|
}
|
|
if((target & BatchIndexer::Target::Content) && !(rebuiltTarget & BatchIndexer::Target::Content)) {
|
|
startTarget |= BatchIndexer::Target::Content;
|
|
m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
|
|
}
|
|
if(target & BatchIndexer::Ocr && !(rebuiltTarget & BatchIndexer::Target::Ocr)) {
|
|
startTarget |= BatchIndexer::Target::Ocr;
|
|
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
|
|
}
|
|
startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, startTarget);
|
|
|
|
//启动监听
|
|
installWatches();
|
|
}
|
|
|
|
BatchIndexer::Targets IndexScheduler::checkAndRebuild(BatchIndexer::Targets target)
|
|
{
|
|
BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Rebuild;
|
|
BatchIndexer::Targets rebuildTarget = BatchIndexer::Target::None;
|
|
if((target & BatchIndexer::Target::Basic) && m_config->isFileIndexEnable() &&
|
|
(m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error
|
|
|| !m_statusRecorder->versionCheck(INDEX_DATABASE_VERSION_KEY, INDEX_DATABASE_VERSION))) {
|
|
qDebug() << "Basic database need rebuild";
|
|
rebuildTarget |= BatchIndexer::Target::Basic;
|
|
m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Initializing);
|
|
}
|
|
if((target & BatchIndexer::Target::Content) && m_config->isContentIndexEnable() &&
|
|
(m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error
|
|
|| !m_statusRecorder->versionCheck(CONTENT_DATABASE_VERSION_KEY, CONTENT_DATABASE_VERSION))) {
|
|
qDebug() << "Content database need rebuild";
|
|
rebuildTarget |= BatchIndexer::Target::Content;
|
|
m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Initializing);
|
|
}
|
|
if((target & BatchIndexer::Target::Ocr) && m_config->isOCREnable() &&
|
|
(m_statusRecorder->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Error
|
|
|| !m_statusRecorder->versionCheck(OCR_CONTENT_DATABASE_VERSION_KEY, OCR_CONTENT_DATABASE_VERSION))) {
|
|
qDebug() << "Ocr content database need rebuild";
|
|
rebuildTarget |= BatchIndexer::Target::Ocr;
|
|
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Initializing);
|
|
}
|
|
startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, rebuildTarget);
|
|
return rebuildTarget;
|
|
}
|
|
|
|
void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList& blackList, BatchIndexer::WorkMode mode, BatchIndexer::Targets target)
|
|
{
|
|
if(BatchIndexer::Target::None != target) {
|
|
if(mode == BatchIndexer::WorkMode::Add) {
|
|
m_addNewPathPendingWorkCount++;
|
|
}
|
|
if(target & BatchIndexer::Basic) {
|
|
m_basicIndexPendingWorkCount++;
|
|
}
|
|
if(target & BatchIndexer::Content) {
|
|
m_contentIndexPendingWorkCount++;
|
|
}
|
|
if(target & BatchIndexer::Ocr) {
|
|
m_ocrContentIndexPendingWorkCount++;
|
|
}
|
|
|
|
m_state = Running;
|
|
Q_EMIT stateChange(m_state);
|
|
|
|
BatchIndexer *indexer = new BatchIndexer(folders, blackList, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop, mode, target);
|
|
connect(indexer, &BatchIndexer::done, this, &IndexScheduler::batchIndexerFinished, Qt::QueuedConnection);
|
|
connect(indexer, &BatchIndexer::progress, this, &IndexScheduler::process, Qt::QueuedConnection);
|
|
connect(indexer, &BatchIndexer::basicIndexDone, this, &IndexScheduler::onBasicIndexDone, Qt::QueuedConnection);
|
|
connect(indexer, &BatchIndexer::contentIndexDone, this, &IndexScheduler::onContentIndexDone, Qt::QueuedConnection);
|
|
connect(indexer, &BatchIndexer::ocrContentIndexDone, this, &IndexScheduler::onOcrContentIndexDone, Qt::QueuedConnection);
|
|
m_threadPool.start(indexer);
|
|
}
|
|
}
|
|
|
|
void IndexScheduler::fileIndexEnable(bool enable)
|
|
{
|
|
if(enable) {
|
|
if(m_basicIndexPendingWorkCount == 0) {
|
|
start(BatchIndexer::Basic);
|
|
} else {
|
|
m_basicIndexStartWaiting = true;
|
|
}
|
|
} else {
|
|
stop(BatchIndexer::Basic);
|
|
}
|
|
}
|
|
|
|
void IndexScheduler::contentIndexEnable(bool enable)
|
|
{
|
|
if(enable) {
|
|
if(m_contentIndexPendingWorkCount == 0) {
|
|
start(BatchIndexer::Content);
|
|
} else {
|
|
m_contentIndexStartWaiting = true;
|
|
}
|
|
} else {
|
|
stop(BatchIndexer::Content);
|
|
}
|
|
}
|
|
|
|
void IndexScheduler::ocrContentIndexEnable(bool enable)
|
|
{
|
|
if(enable) {
|
|
if(m_ocrContentIndexPendingWorkCount == 0) {
|
|
start(BatchIndexer::Ocr);
|
|
} else {
|
|
m_ocrContentIndexStartWaiting = true;
|
|
}
|
|
} else {
|
|
stop(BatchIndexer::Ocr);
|
|
}
|
|
}
|
|
|
|
void IndexScheduler::updateIndex(const QVector<PendingFile> &files)
|
|
{
|
|
qDebug() << "updateIndex=====";
|
|
m_updatePendingWorkCount++;
|
|
|
|
m_state = Running;
|
|
Q_EMIT stateChange(m_state);
|
|
|
|
auto *updateJob = new IndexUpdater(files, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop);
|
|
connect(updateJob, &IndexUpdater::done, this, &IndexScheduler::updateFinished, Qt::QueuedConnection);
|
|
m_threadPool.start(updateJob);
|
|
}
|
|
|
|
void IndexScheduler::batchIndexerFinished(BatchIndexer::WorkMode mode, BatchIndexer::Targets targets)
|
|
{
|
|
if(mode == BatchIndexer::WorkMode::Add) {
|
|
m_addNewPathPendingWorkCount--;
|
|
}
|
|
if(isIdle()) {
|
|
m_state = Idle;
|
|
Q_EMIT stateChange(m_state);
|
|
}
|
|
}
|
|
|
|
void IndexScheduler::updateFinished()
|
|
{
|
|
m_updatePendingWorkCount--;
|
|
if(isIdle()) {
|
|
m_state = Idle;
|
|
Q_EMIT stateChange(m_state);
|
|
}
|
|
}
|
|
|
|
bool IndexScheduler::isIdle()
|
|
{
|
|
return m_basicIndexPendingWorkCount == 0
|
|
&& m_contentIndexPendingWorkCount == 0
|
|
&& m_ocrContentIndexPendingWorkCount == 0
|
|
&& m_updatePendingWorkCount == 0
|
|
&& m_addNewPathPendingWorkCount == 0;
|
|
}
|
|
|
|
void IndexScheduler::onBasicIndexDone(BatchIndexer::WorkMode mode)
|
|
{
|
|
Q_UNUSED(mode)
|
|
m_basicIndexPendingWorkCount--;
|
|
|
|
bool success = false;
|
|
if(m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) {
|
|
m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
|
|
success = true;
|
|
}
|
|
Q_EMIT basicIndexDone(success);
|
|
checkIfStartsWaiting();
|
|
}
|
|
|
|
void IndexScheduler::onContentIndexDone(BatchIndexer::WorkMode mode)
|
|
{
|
|
Q_UNUSED(mode)
|
|
m_contentIndexPendingWorkCount--;
|
|
|
|
bool success = false;
|
|
if(m_statusRecorder->getStatus(CONTENT_INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) {
|
|
m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
|
|
success = true;
|
|
}
|
|
Q_EMIT contentIndexDone(success);
|
|
checkIfStartsWaiting();
|
|
}
|
|
|
|
void IndexScheduler::onOcrContentIndexDone(BatchIndexer::WorkMode mode)
|
|
{
|
|
Q_UNUSED(mode)
|
|
m_ocrContentIndexPendingWorkCount--;
|
|
|
|
bool success = false;
|
|
if(m_statusRecorder->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) {
|
|
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
|
|
success = true;
|
|
}
|
|
Q_EMIT contentIndexDone(success);
|
|
checkIfStartsWaiting();
|
|
}
|
|
|
|
void IndexScheduler::checkIfStartsWaiting()
|
|
{
|
|
BatchIndexer::Targets targets = BatchIndexer::None;
|
|
if(m_basicIndexStartWaiting && !m_basicIndexPendingWorkCount) {
|
|
targets |= BatchIndexer::Basic;
|
|
m_basicIndexStartWaiting = false;
|
|
}
|
|
if(m_contentIndexStartWaiting && !m_contentIndexPendingWorkCount) {
|
|
targets |= BatchIndexer::Content;
|
|
m_contentIndexStartWaiting = false;
|
|
}
|
|
if(m_ocrContentIndexStartWaiting && !m_ocrContentIndexPendingWorkCount) {
|
|
targets |= BatchIndexer::Ocr;
|
|
m_ocrContentIndexStartWaiting = false;
|
|
}
|
|
if(targets != BatchIndexer::None) {
|
|
start(targets);
|
|
}
|
|
}
|
|
|
|
void IndexScheduler::installWatches()
|
|
{
|
|
if(m_config->isFileIndexEnable() || m_config->isContentIndexEnable() || m_config->isOCREnable()) {
|
|
if(!m_watchInstalled) {
|
|
m_fileWatcher.installWatches();
|
|
m_watchInstalled = true;
|
|
}
|
|
}
|
|
}
|