增加文本内容解析配置功能和强制执行增量更新功能

This commit is contained in:
iaom 2024-01-23 11:18:55 +08:00
parent a6fb658cd6
commit fb8b0b6756
12 changed files with 239 additions and 110 deletions

View File

@ -37,47 +37,7 @@ static const QStringList allAppPath = {
{"/usr/share/applications/"} {"/usr/share/applications/"}
}; };
static const QMap<QString, bool> targetFileTypeMap = {
{"doc", true},
{"docx", true},
{"ppt", true},
{"pptx", true},
{"xls", true},
{"xlsx", true},
{"txt", true},
{"dot", true},
{"wps", true},
{"pps", true},
{"dps", true},
{"et", true},
{"pdf", true},
{"html", true},
{"uof", true},
{"uot", true},
{"uos", true},
{"uop", true},
{"ofd", true}
};
static const QMap<QString, bool> targetPhotographTypeMap = {
{"png", true},
{"bmp", true},
{"hdr", false},
{"gif", true},
{"tif", true},
{"tiff", true},
{"heif", false},
{"webp", true},
{"jpe", true},
{"dib", false},
{"psd", false},
{"jng", false},
{"xpm", false},//pix read error.
{"j2k", false},
{"jp2", false},
{"jpg", true},
{"jpeg", true} // TODO 待完善,后续改为配置文件
};
//TODO Put things that needed to be put here here. //TODO Put things that needed to be put here here.
/** /**
* @brief The DataBaseType enum * @brief The DataBaseType enum

View File

@ -40,6 +40,7 @@
#include "hanzi-to-pinyin.h" #include "hanzi-to-pinyin.h"
#include "common.h" #include "common.h"
#include "icon-loader.h" #include "icon-loader.h"
#include "file-indexer-config.h"
using namespace UkuiSearch; using namespace UkuiSearch;
@ -482,7 +483,7 @@ bool FileUtils::isEncrypedOrUnsupport(const QString& path, const QString& suffix
} }
return true; return true;
} else if (true == targetPhotographTypeMap[suffix]) { } else if (FileIndexerConfig::getInstance()->ocrContentIndexTarget()[suffix]) {
return !isOcrSupportSize(path); return !isOcrSupportSize(path);
} else { } else {
// qInfo() << "Unsupport format:[" << path << "][" << type.name() << "]"; // qInfo() << "Unsupport format:[" << path << "][" << type.name() << "]";

View File

@ -24,6 +24,7 @@
#include <malloc.h> #include <malloc.h>
#include <QQueue> #include <QQueue>
#include <QDateTime> #include <QDateTime>
#include <QMetaEnum>
#include "file-utils.h" #include "file-utils.h"
#include "basic-indexer.h" #include "basic-indexer.h"
@ -49,10 +50,13 @@ void BatchIndexer::run()
{ {
QElapsedTimer timer; QElapsedTimer timer;
timer.start(); timer.start();
if(m_target == Target::None || (m_indexStop->LOAD && m_contentIndexStop->LOAD)) { if((!m_target.testFlag(Target::Basic) || m_indexStop->LOAD)
&& (!m_target.testFlag(Target::Content) || m_contentIndexStop->LOAD)
&& (!m_target.testFlag(Target::Ocr) || m_contentIndexOcrStop->LOAD)) {
Q_EMIT done(m_mode, m_target); Q_EMIT done(m_mode, m_target);
return; return;
} }
fetch(); fetch();
if(m_target & Target::Basic) { if(m_target & Target::Basic) {
@ -69,7 +73,8 @@ void BatchIndexer::run()
} }
m_cache.clear(); m_cache.clear();
malloc_trim(0); malloc_trim(0);
qDebug() << "FirstRunIndexer: time :" << timer.elapsed() << "milliseconds";
qDebug() << "BatchIndexer " << QMetaEnum::fromType<Targets>().valueToKeys(m_target) << ": time :" << timer.elapsed() << "milliseconds";
Q_EMIT done(m_mode, m_target); Q_EMIT done(m_mode, m_target);
} }
@ -214,7 +219,7 @@ void BatchIndexer::contentIndex()
if(m_mode == WorkMode::Rebuild || m_mode == WorkMode::Add) { if(m_mode == WorkMode::Rebuild || m_mode == WorkMode::Add) {
for(const QString& path : m_cache) { for(const QString& path : m_cache) {
info.setFile(path); info.setFile(path);
if(targetFileTypeMap[info.suffix()] && info.isFile()) { if(FileIndexerConfig::getInstance()->contentIndexTarget()[info.suffix()] && info.isFile()) {
if(!FileUtils::isEncrypedOrUnsupport(path, info.suffix())) { if(!FileUtils::isEncrypedOrUnsupport(path, info.suffix())) {
filesNeedIndex.append(path); filesNeedIndex.append(path);
} }
@ -225,7 +230,7 @@ void BatchIndexer::contentIndex()
qDebug() << indexTimes.size() << "documents recorded"; qDebug() << indexTimes.size() << "documents recorded";
for(const QString& path : m_cache) { for(const QString& path : m_cache) {
info.setFile(path); info.setFile(path);
if(targetFileTypeMap[info.suffix()] && info.isFile()) { if(FileIndexerConfig::getInstance()->contentIndexTarget()[info.suffix()] && info.isFile()) {
std::string uterm = FileUtils::makeDocUterm(path); std::string uterm = FileUtils::makeDocUterm(path);
if(indexTimes.value(uterm) != info.lastModified().toString("yyyyMMddHHmmsszzz").toStdString()) { if(indexTimes.value(uterm) != info.lastModified().toString("yyyyMMddHHmmsszzz").toStdString()) {
if(!FileUtils::isEncrypedOrUnsupport(path, info.suffix())) { if(!FileUtils::isEncrypedOrUnsupport(path, info.suffix())) {
@ -305,7 +310,7 @@ void BatchIndexer::ocrIndex()
if(m_mode == WorkMode::Rebuild || m_mode == WorkMode::Add) { if(m_mode == WorkMode::Rebuild || m_mode == WorkMode::Add) {
for(const QString &path : m_cache) { for(const QString &path : m_cache) {
info.setFile(path); info.setFile(path);
if(targetPhotographTypeMap[info.suffix()] && info.isFile()) { if(FileIndexerConfig::getInstance()->ocrContentIndexTarget()[info.suffix()] && info.isFile()) {
if(!FileUtils::isEncrypedOrUnsupport(path, info.suffix())) { if(!FileUtils::isEncrypedOrUnsupport(path, info.suffix())) {
filesNeedOCRIndex.append(path); filesNeedOCRIndex.append(path);
} }
@ -316,7 +321,7 @@ void BatchIndexer::ocrIndex()
qDebug() << indexTimes.size() << "documents recorded"; qDebug() << indexTimes.size() << "documents recorded";
for(const QString& path : m_cache) { for(const QString& path : m_cache) {
info.setFile(path); info.setFile(path);
if(targetPhotographTypeMap[info.suffix()] && info.isFile()) { if(FileIndexerConfig::getInstance()->ocrContentIndexTarget()[info.suffix()] && info.isFile()) {
std::string uterm = FileUtils::makeDocUterm(path); std::string uterm = FileUtils::makeDocUterm(path);
if(indexTimes.value(uterm) != info.lastModified().toString("yyyyMMddHHmmsszzz").toStdString()) { if(indexTimes.value(uterm) != info.lastModified().toString("yyyyMMddHHmmsszzz").toStdString()) {
if(!FileUtils::isEncrypedOrUnsupport(path, info.suffix())) { if(!FileUtils::isEncrypedOrUnsupport(path, info.suffix())) {

View File

@ -54,9 +54,11 @@ public:
Basic = 1u << 0, Basic = 1u << 0,
Content = 1u << 1, Content = 1u << 1,
Ocr = 1u << 2, Ocr = 1u << 2,
All = Basic | Content All = Basic | Content | Ocr
}; };
Q_ENUM(Target)
Q_DECLARE_FLAGS(Targets, Target) Q_DECLARE_FLAGS(Targets, Target)
Q_FLAG(Targets)
BatchIndexer(const QStringList& folders, BatchIndexer(const QStringList& folders,
const QStringList& blackList, const QStringList& blackList,

View File

@ -17,12 +17,15 @@
* Authors: iaom <zhangpengfei@kylinos.cn> * Authors: iaom <zhangpengfei@kylinos.cn>
* *
*/ */
#include "file-indexer-config.h"\ #include "file-indexer-config.h"
#include <mutex> #include <mutex>
#include <QDebug> #include <QDebug>
#include <QDir> #include <QDir>
#define INDEX_SETTINGS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-service.conf"
static const QString INDEX_SETTINGS = QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-service.conf";
static const QString CONTENT_INDEX_TARGET_TYPE = "contentIndexTarget";
static const QString OCR_CONTENT_INDEX_TARGET_TYPE = "ocrContentIndexTarget";
/** /**
* changelog: 1.1 ocr开关 * changelog: 1.1 ocr开关
*/ */
@ -32,19 +35,18 @@ static const QString FILE_INDEX_ENABLE_KEY = QStringLiteral("fileIndexEnable");
static const QString CONTENT_INDEX_ENABLE_KEY = QStringLiteral("contentIndexEnable"); static const QString CONTENT_INDEX_ENABLE_KEY = QStringLiteral("contentIndexEnable");
static const QString CONTENT_FUZZY_SEARCH_KEY = QStringLiteral("contentFuzzySearch"); static const QString CONTENT_FUZZY_SEARCH_KEY = QStringLiteral("contentFuzzySearch");
static const QString CONTENT_INDEX_ENABLE_OCR_KEY = QStringLiteral("contentIndexEnableOcr"); static const QString CONTENT_INDEX_ENABLE_OCR_KEY = QStringLiteral("contentIndexEnableOcr");
static const QString OCR_ENABLE_KEY = QStringLiteral("ocrEnable");
static const QString META_DATA_INDEX_ENABLE_KEY = QStringLiteral("metaDataIndexEnable"); static const QString META_DATA_INDEX_ENABLE_KEY = QStringLiteral("metaDataIndexEnable");
static const QString CONFIG_VERSION_KEY = QStringLiteral("version"); static const QString CONFIG_VERSION_KEY = QStringLiteral("version");
static std::once_flag flag; static std::once_flag flag;
static FileIndexerConfig *global_intance = nullptr; static UkuiSearch::FileIndexerConfig *globalInstance = nullptr;
using namespace UkuiSearch;
FileIndexerConfig *FileIndexerConfig::getInstance() FileIndexerConfig *FileIndexerConfig::getInstance()
{ {
std::call_once(flag, [ & ] { std::call_once(flag, [ & ] {
global_intance = new FileIndexerConfig(); globalInstance = new FileIndexerConfig();
}); });
return global_intance; return globalInstance;
} }
FileIndexerConfig::FileIndexerConfig(QObject *parent) FileIndexerConfig::FileIndexerConfig(QObject *parent)
@ -99,14 +101,11 @@ FileIndexerConfig::FileIndexerConfig(QObject *parent)
} else { } else {
qWarning() << UKUI_SEARCH_SCHEMAS << " is not found!"; qWarning() << UKUI_SEARCH_SCHEMAS << " is not found!";
} }
m_settings = new QSettings(INDEX_SETTINGS, QSettings::IniFormat, this); m_settings = new QSettings(INDEX_SETTINGS, QSettings::IniFormat, this);
} }
FileIndexerConfig::~FileIndexerConfig() FileIndexerConfig::~FileIndexerConfig()
{ = default;
}
QStringList FileIndexerConfig::currentIndexableDir() QStringList FileIndexerConfig::currentIndexableDir()
{ {
@ -182,3 +181,42 @@ bool FileIndexerConfig::isMetaDataIndexEnable()
{ {
return m_settings->value(META_DATA_INDEX_ENABLE_KEY, true).toBool(); return m_settings->value(META_DATA_INDEX_ENABLE_KEY, true).toBool();
} }
QMap<QString, bool> FileIndexerConfig::contentIndexTarget()
{
return m_targetFileTypeMap;
}
QMap<QString, bool> FileIndexerConfig::ocrContentIndexTarget()
{
return m_targetPhotographTypeMap;
}
void FileIndexerConfig::sync()
{
m_settings->sync();
m_settings->beginGroup(CONTENT_INDEX_TARGET_TYPE);
if(m_settings->allKeys().isEmpty()) {
for(const QString &type : m_targetFileTypeMap.keys()) {
m_settings->setValue(type, m_targetFileTypeMap[type]);
}
} else {
m_targetFileTypeMap.clear();
for(const QString &type : m_settings->allKeys()) {
m_targetFileTypeMap.insert(type, m_settings->value(type).toBool());
}
}
m_settings->endGroup();
m_settings->beginGroup(OCR_CONTENT_INDEX_TARGET_TYPE);
if(m_settings->allKeys().isEmpty()) {
for(const QString &type : m_targetPhotographTypeMap.keys()) {
m_settings->setValue(type, m_targetPhotographTypeMap[type]);
}
} else {
m_targetPhotographTypeMap.clear();
for(const QString &type : m_settings->allKeys()) {
m_targetPhotographTypeMap.insert(type, m_settings->value(type).toBool());
}
}
m_settings->endGroup();
}

View File

@ -24,6 +24,7 @@
#include <QSettings> #include <QSettings>
#include <QGSettings> #include <QGSettings>
#include "dir-watcher.h" #include "dir-watcher.h"
namespace UkuiSearch{
class FileIndexerConfig : public QObject class FileIndexerConfig : public QObject
{ {
@ -65,6 +66,9 @@ public:
* @return * @return
*/ */
bool isMetaDataIndexEnable(); bool isMetaDataIndexEnable();
QMap<QString, bool> contentIndexTarget();
QMap<QString, bool> ocrContentIndexTarget();
void sync();
Q_SIGNALS: Q_SIGNALS:
/** /**
@ -96,11 +100,52 @@ Q_SIGNALS:
private: private:
explicit FileIndexerConfig(QObject *parent = nullptr); explicit FileIndexerConfig(QObject *parent = nullptr);
~FileIndexerConfig(); ~FileIndexerConfig();
QMap<QString, bool> m_targetFileTypeMap = {
{"doc", true},
{"docx", true},
{"ppt", true},
{"pptx", true},
{"xls", true},
{"xlsx", true},
{"txt", true},
{"dot", true},
{"wps", true},
{"pps", true},
{"dps", true},
{"et", true},
{"pdf", true},
{"html", true},
{"uof", true},
{"uot", true},
{"uos", true},
{"uop", true},
{"ofd", true}
};
QMap<QString, bool> m_targetPhotographTypeMap = {
{"png", true},
{"bmp", true},
{"hdr", false},
{"gif", true},
{"tif", true},
{"tiff", true},
{"heif", false},
{"webp", true},
{"jpe", true},
{"dib", false},
{"psd", false},
{"jng", false},
{"xpm", false},//pix read error.
{"j2k", false},
{"jp2", false},
{"jpg", true},
{"jpeg", true}
};
DirWatcher *m_dirWatcher = nullptr; DirWatcher *m_dirWatcher = nullptr;
QGSettings *m_gsettings = nullptr; QGSettings *m_gsettings = nullptr;
QSettings *m_settings = nullptr; QSettings *m_settings = nullptr;
}; };
}
#endif // FILEINDEXERCONFIG_H #endif // FILEINDEXERCONFIG_H

View File

@ -23,6 +23,7 @@
#include <mutex> #include <mutex>
#include "file-extraction-result.h" #include "file-extraction-result.h"
#include "common.h" #include "common.h"
#include "file-indexer-config.h"
using namespace UkuiSearch; using namespace UkuiSearch;
FileReader *g_instance = nullptr; FileReader *g_instance = nullptr;
@ -39,7 +40,7 @@ FileReader::FileReader()
= default; = default;
void FileReader::getTextContent(const QString &path, QString &textContent, const QString &suffix) void FileReader::getTextContent(const QString &path, QString &textContent, const QString &suffix)
{ {
if(targetPhotographTypeMap[suffix]) { if(FileIndexerConfig::getInstance()->ocrContentIndexTarget()[suffix]) {
textContent = UkuiFileMetadata::OcrUtils::getTextInPicture(path); textContent = UkuiFileMetadata::OcrUtils::getTextInPicture(path);
return; return;
} }

View File

@ -18,6 +18,8 @@
* *
*/ */
#include "index-scheduler.h" #include "index-scheduler.h"
#include <QCoreApplication>
#include <QMetaEnum>
#include "index-updater.h" #include "index-updater.h"
#include "compatible-define.h" #include "compatible-define.h"
@ -52,18 +54,20 @@ IndexScheduler::IndexScheduler(QObject *parent) :
} }
if(m_config->isContentIndexEnable()) { if(m_config->isContentIndexEnable()) {
targets |= BatchIndexer::Target::Content; targets |= BatchIndexer::Target::Content;
if(m_config->isOCREnable()) {
targets |= BatchIndexer::Target::Ocr;
}
} else { } else {
m_contentIndexStop.fetchAndStoreRelaxed(1); m_contentIndexStop.fetchAndStoreRelaxed(1);
} }
if(m_config->isOCREnable()) {
targets |= BatchIndexer::Target::Ocr;
} else {
m_ocrContentIndexStop.fetchAndStoreRelaxed(1);
}
start(targets); start(targets);
} }
void IndexScheduler::addNewPath(const QString &folders, const QStringList &blackList) void IndexScheduler::addNewPath(const QString &folders, const QStringList &blackList)
{ {
if(m_indexStop.LOAD && m_contentIndexStop.LOAD) { if(m_indexStop.LOAD && m_contentIndexStop.LOAD && m_ocrContentIndexStop.LOAD) {
qDebug() << "Index Scheduler is being stopped, add operation will be executed when started up next time."; qDebug() << "Index Scheduler is being stopped, add operation will be executed when started up next time.";
return; return;
} }
@ -90,7 +94,7 @@ void IndexScheduler::addNewPath(const QString &folders, const QStringList &black
void IndexScheduler::removeIndex(const QString &folders) void IndexScheduler::removeIndex(const QString &folders)
{ {
if(m_indexStop.LOAD && m_contentIndexStop.LOAD) { if(m_indexStop.LOAD && m_contentIndexStop.LOAD && m_ocrContentIndexStop.LOAD) {
qDebug() << "Index Scheduler is being stopped, remove operation will be executed when started up next time."; qDebug() << "Index Scheduler is being stopped, remove operation will be executed when started up next time.";
return; return;
} }
@ -129,35 +133,24 @@ IndexScheduler::IndexerState IndexScheduler::getIndexState()
return m_state; return m_state;
} }
void IndexScheduler::start(BatchIndexer::Targets target) void IndexScheduler::forceUpdate(BatchIndexer::Targets target)
{ {
qDebug() << "Index scheduler start." << target; m_config->sync();
stop(target);
BatchIndexer::Targets realTargets = BatchIndexer::Target::None;
//检查是否有任务未完成
if(target & BatchIndexer::Basic) { if(target & BatchIndexer::Basic) {
if(m_indexPendingWorkCount == 0) { fileIndexEnable(true);
realTargets |= BatchIndexer::Target::Basic;
}
} }
if(target & BatchIndexer::Content) { if(target & BatchIndexer::Content) {
if(m_contentIndexPendingWorkCount == 0) { contentIndexEnable(true);
realTargets |= BatchIndexer::Target::Content;
}
} }
if(target & BatchIndexer::Ocr) { if(target & BatchIndexer::Ocr) {
if(m_ocrContentIndexPendingWorkCount == 0) { ocrContentIndexEnable(true);
realTargets |= BatchIndexer::Target::Ocr;
}
} }
}
if(realTargets == BatchIndexer::Target::None) { void IndexScheduler::start(BatchIndexer::Targets target)
qDebug() << "Index scheduler running, start operation ignored.\n" {
<< "index pending work count: " << m_contentIndexPendingWorkCount << "\n" qDebug() << "Index scheduler start." << QMetaEnum::fromType<BatchIndexer::Targets>().valueToKeys(target) ;
<< "Content index pending work count: " << m_contentIndexPendingWorkCount << "\n"
<< "Ocr content index pending work count: " << m_ocrContentIndexPendingWorkCount << "\n";
return;
}
//打开异步控制开关 //打开异步控制开关
if(target & BatchIndexer::Basic) { if(target & BatchIndexer::Basic) {
@ -169,30 +162,33 @@ void IndexScheduler::start(BatchIndexer::Targets target)
if(target & BatchIndexer::Ocr) { if(target & BatchIndexer::Ocr) {
m_ocrContentIndexStop.fetchAndStoreRelaxed(0); m_ocrContentIndexStop.fetchAndStoreRelaxed(0);
} }
if(target == BatchIndexer::None) {
return;
}
//检查是否有数据库需要重建并且执行重建 //检查是否有数据库需要重建并且执行重建
BatchIndexer::Targets rebuiltTarget = checkAndRebuild(realTargets); BatchIndexer::Targets rebuiltTarget = checkAndRebuild(target);
BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Update; BatchIndexer::WorkMode mode = BatchIndexer::WorkMode::Update;
BatchIndexer::Targets startTarget = BatchIndexer::Target::None; BatchIndexer::Targets startTarget = BatchIndexer::Target::None;
//如果数据库被执行过重建,那么跳过增量更新步骤。 //如果数据库被执行过重建,那么跳过增量更新步骤。
if((realTargets & BatchIndexer::Target::Basic) && !(rebuiltTarget & BatchIndexer::Target::Basic)) { if((target & BatchIndexer::Target::Basic) && !(rebuiltTarget & BatchIndexer::Target::Basic)) {
startTarget |= BatchIndexer::Target::Basic; startTarget |= BatchIndexer::Target::Basic;
m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating); m_statusRecorder->setStatus(INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
} }
if((realTargets & BatchIndexer::Target::Content) && !(rebuiltTarget & BatchIndexer::Target::Content)) { if((target & BatchIndexer::Target::Content) && !(rebuiltTarget & BatchIndexer::Target::Content)) {
startTarget |= BatchIndexer::Target::Content; startTarget |= BatchIndexer::Target::Content;
m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating); m_statusRecorder->setStatus(CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
} }
if(realTargets & BatchIndexer::Ocr && !(rebuiltTarget & BatchIndexer::Target::Ocr)) { if(target & BatchIndexer::Ocr && !(rebuiltTarget & BatchIndexer::Target::Ocr)) {
startTarget |= BatchIndexer::Target::Ocr; startTarget |= BatchIndexer::Target::Ocr;
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating); m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
} }
startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, startTarget); startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, startTarget);
//启动监听 //启动监听
m_fileWatcher.installWatches(); installWatches();
} }
BatchIndexer::Targets IndexScheduler::checkAndRebuild(BatchIndexer::Targets target) BatchIndexer::Targets IndexScheduler::checkAndRebuild(BatchIndexer::Targets target)
@ -231,7 +227,7 @@ void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList&
m_addNewPathPendingWorkCount++; m_addNewPathPendingWorkCount++;
} }
if(target & BatchIndexer::Basic) { if(target & BatchIndexer::Basic) {
m_indexPendingWorkCount++; m_basicIndexPendingWorkCount++;
} }
if(target & BatchIndexer::Content) { if(target & BatchIndexer::Content) {
m_contentIndexPendingWorkCount++; m_contentIndexPendingWorkCount++;
@ -256,7 +252,11 @@ void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList&
void IndexScheduler::fileIndexEnable(bool enable) void IndexScheduler::fileIndexEnable(bool enable)
{ {
if(enable) { if(enable) {
start(BatchIndexer::Basic); if(m_basicIndexPendingWorkCount == 0) {
start(BatchIndexer::Basic);
} else {
m_basicIndexStartWaiting = true;
}
} else { } else {
stop(BatchIndexer::Basic); stop(BatchIndexer::Basic);
} }
@ -265,7 +265,11 @@ void IndexScheduler::fileIndexEnable(bool enable)
void IndexScheduler::contentIndexEnable(bool enable) void IndexScheduler::contentIndexEnable(bool enable)
{ {
if(enable) { if(enable) {
start(BatchIndexer::Content); if(m_contentIndexPendingWorkCount == 0) {
start(BatchIndexer::Content);
} else {
m_contentIndexStartWaiting = true;
}
} else { } else {
stop(BatchIndexer::Content); stop(BatchIndexer::Content);
} }
@ -274,7 +278,11 @@ void IndexScheduler::contentIndexEnable(bool enable)
void IndexScheduler::ocrContentIndexEnable(bool enable) void IndexScheduler::ocrContentIndexEnable(bool enable)
{ {
if(enable) { if(enable) {
start(BatchIndexer::Ocr); if(m_ocrContentIndexPendingWorkCount == 0) {
start(BatchIndexer::Ocr);
} else {
m_ocrContentIndexStartWaiting = true;
}
} else { } else {
stop(BatchIndexer::Ocr); stop(BatchIndexer::Ocr);
} }
@ -288,7 +296,7 @@ void IndexScheduler::updateIndex(const QVector<PendingFile> &files)
m_state = Running; m_state = Running;
Q_EMIT stateChange(m_state); Q_EMIT stateChange(m_state);
IndexUpdater *updateJob = new IndexUpdater(files, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop); auto *updateJob = new IndexUpdater(files, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop);
connect(updateJob, &IndexUpdater::done, this, &IndexScheduler::updateFinished, Qt::QueuedConnection); connect(updateJob, &IndexUpdater::done, this, &IndexScheduler::updateFinished, Qt::QueuedConnection);
m_threadPool.start(updateJob); m_threadPool.start(updateJob);
} }
@ -315,7 +323,7 @@ void IndexScheduler::updateFinished()
bool IndexScheduler::isIdle() bool IndexScheduler::isIdle()
{ {
return m_indexPendingWorkCount == 0 return m_basicIndexPendingWorkCount == 0
&& m_contentIndexPendingWorkCount == 0 && m_contentIndexPendingWorkCount == 0
&& m_ocrContentIndexPendingWorkCount == 0 && m_ocrContentIndexPendingWorkCount == 0
&& m_updatePendingWorkCount == 0 && m_updatePendingWorkCount == 0
@ -325,7 +333,7 @@ bool IndexScheduler::isIdle()
void IndexScheduler::onBasicIndexDone(BatchIndexer::WorkMode mode) void IndexScheduler::onBasicIndexDone(BatchIndexer::WorkMode mode)
{ {
Q_UNUSED(mode) Q_UNUSED(mode)
m_indexPendingWorkCount--; m_basicIndexPendingWorkCount--;
bool success = false; bool success = false;
if(m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) { if(m_statusRecorder->getStatus(INDEX_DATABASE_STATE_KEY).toInt() != IndexStatusRecorder::State::Error) {
@ -333,6 +341,7 @@ void IndexScheduler::onBasicIndexDone(BatchIndexer::WorkMode mode)
success = true; success = true;
} }
Q_EMIT basicIndexDone(success); Q_EMIT basicIndexDone(success);
checkIfStartsWaiting();
} }
void IndexScheduler::onContentIndexDone(BatchIndexer::WorkMode mode) void IndexScheduler::onContentIndexDone(BatchIndexer::WorkMode mode)
@ -346,6 +355,7 @@ void IndexScheduler::onContentIndexDone(BatchIndexer::WorkMode mode)
success = true; success = true;
} }
Q_EMIT contentIndexDone(success); Q_EMIT contentIndexDone(success);
checkIfStartsWaiting();
} }
void IndexScheduler::onOcrContentIndexDone(BatchIndexer::WorkMode mode) void IndexScheduler::onOcrContentIndexDone(BatchIndexer::WorkMode mode)
@ -359,4 +369,35 @@ void IndexScheduler::onOcrContentIndexDone(BatchIndexer::WorkMode mode)
success = true; success = true;
} }
Q_EMIT contentIndexDone(success); Q_EMIT contentIndexDone(success);
checkIfStartsWaiting();
}
void IndexScheduler::checkIfStartsWaiting()
{
BatchIndexer::Targets targets = BatchIndexer::None;
if(m_basicIndexStartWaiting && !m_basicIndexPendingWorkCount) {
targets |= BatchIndexer::Basic;
m_basicIndexStartWaiting = false;
}
if(m_contentIndexStartWaiting && !m_contentIndexPendingWorkCount) {
targets |= BatchIndexer::Content;
m_contentIndexStartWaiting = false;
}
if(m_ocrContentIndexStartWaiting && !m_ocrContentIndexPendingWorkCount) {
targets |= BatchIndexer::Ocr;
m_ocrContentIndexStartWaiting = false;
}
if(targets != BatchIndexer::None) {
start(targets);
}
}
void IndexScheduler::installWatches()
{
if(m_config->isFileIndexEnable() || m_config->isContentIndexEnable() || m_config->isOCREnable()) {
if(!m_watchInstalled) {
m_fileWatcher.installWatches();
m_watchInstalled = true;
}
}
} }

View File

@ -23,6 +23,7 @@
#include <QObject> #include <QObject>
#include <QThreadPool> #include <QThreadPool>
#include <QAtomicInt> #include <QAtomicInt>
#include <QEvent>
#include "file-watcher.h" #include "file-watcher.h"
#include "index-status-recorder.h" #include "index-status-recorder.h"
#include "common.h" #include "common.h"
@ -44,7 +45,12 @@ public:
explicit IndexScheduler(QObject *parent = nullptr); explicit IndexScheduler(QObject *parent = nullptr);
Q_SCRIPTABLE IndexerState getIndexState(); Q_INVOKABLE IndexerState getIndexState();
/**
*
* @param target
*/
Q_INVOKABLE void forceUpdate(BatchIndexer::Targets target);
Q_SIGNALS: Q_SIGNALS:
void stateChange(IndexerState); void stateChange(IndexerState);
@ -59,12 +65,12 @@ private Q_SLOTS:
* @param folders * @param folders
* @param blackList * @param blackList
*/ */
Q_SCRIPTABLE void addNewPath(const QString &folders, const QStringList& blackList = QStringList()); void addNewPath(const QString &folders, const QStringList& blackList = QStringList());
/** /**
* @brief removeIndex * @brief removeIndex
* @param folders * @param folders
*/ */
Q_SCRIPTABLE void removeIndex(const QString& folders); void removeIndex(const QString& folders);
void start(BatchIndexer::Targets target); void start(BatchIndexer::Targets target);
void stop(BatchIndexer::Targets target); void stop(BatchIndexer::Targets target);
void fileIndexEnable(bool enable); void fileIndexEnable(bool enable);
@ -77,6 +83,8 @@ private Q_SLOTS:
void onBasicIndexDone(BatchIndexer::WorkMode mode); void onBasicIndexDone(BatchIndexer::WorkMode mode);
void onContentIndexDone(BatchIndexer::WorkMode mode); void onContentIndexDone(BatchIndexer::WorkMode mode);
void onOcrContentIndexDone(BatchIndexer::WorkMode mode); void onOcrContentIndexDone(BatchIndexer::WorkMode mode);
void checkIfStartsWaiting();
void installWatches();
private: private:
/** /**
@ -95,12 +103,21 @@ private:
QAtomicInt m_ocrContentIndexStop; QAtomicInt m_ocrContentIndexStop;
QThreadPool m_threadPool; QThreadPool m_threadPool;
quint64 m_indexPendingWorkCount = 0; //等待完成的工作数量
quint64 m_basicIndexPendingWorkCount = 0;
quint64 m_contentIndexPendingWorkCount = 0; quint64 m_contentIndexPendingWorkCount = 0;
quint64 m_ocrContentIndexPendingWorkCount= 0; quint64 m_ocrContentIndexPendingWorkCount= 0;
quint64 m_updatePendingWorkCount = 0; quint64 m_updatePendingWorkCount = 0;
quint64 m_addNewPathPendingWorkCount = 0; quint64 m_addNewPathPendingWorkCount = 0;
//是否有启动操作正在等待已有工作结束
bool m_basicIndexStartWaiting = false;
bool m_contentIndexStartWaiting = false;
bool m_ocrContentIndexStartWaiting = false;
//监听是否已安装
bool m_watchInstalled = false;
}; };
} }
#endif // INDEXSCHEDULER_H #endif // INDEXSCHEDULER_H

View File

@ -98,10 +98,10 @@ void IndexUpdater::updateContentIndex()
qDebug() << "| remove:" <<file.path(); qDebug() << "| remove:" <<file.path();
if(file.isDir()) { if(file.isDir()) {
contentDb.removeChildrenDocument(file.path()); contentDb.removeChildrenDocument(file.path());
} else if(targetFileTypeMap[suffix]) { } else if(FileIndexerConfig::getInstance()->contentIndexTarget()[suffix]) {
contentDb.removeDocument(file.path()); contentDb.removeDocument(file.path());
} }
} else if(targetFileTypeMap[suffix] && !file.isDir()) { } else if(FileIndexerConfig::getInstance()->contentIndexTarget()[suffix] && !file.isDir()) {
if(FileUtils::isEncrypedOrUnsupport(file.path(), suffix)) { if(FileUtils::isEncrypedOrUnsupport(file.path(), suffix)) {
if(file.isModified() || file.isMoveTo()) { if(file.isModified() || file.isMoveTo()) {
contentDb.removeDocument(file.path()); contentDb.removeDocument(file.path());
@ -149,10 +149,10 @@ void IndexUpdater::updateOcrContentIndex()
qDebug() << "| remove:" <<file.path(); qDebug() << "| remove:" <<file.path();
if(file.isDir()) { if(file.isDir()) {
contentDb.removeChildrenDocument(file.path()); contentDb.removeChildrenDocument(file.path());
} else if(targetPhotographTypeMap[suffix]) { } else if(FileIndexerConfig::getInstance()->ocrContentIndexTarget()[suffix]) {
contentDb.removeDocument(file.path()); contentDb.removeDocument(file.path());
} }
} else if(targetPhotographTypeMap[suffix] && !file.isDir()) { } else if(FileIndexerConfig::getInstance()->ocrContentIndexTarget()[suffix] && !file.isDir()) {
if(FileUtils::isEncrypedOrUnsupport(file.path(), suffix)) { if(FileUtils::isEncrypedOrUnsupport(file.path(), suffix)) {
if(file.isModified() || file.isMoveTo()) { if(file.isModified() || file.isMoveTo()) {
contentDb.removeDocument(file.path()); contentDb.removeDocument(file.path());

View File

@ -71,7 +71,7 @@ bool SearchManager::creatResultInfo(SearchPluginIface::ResultInfo &ri, const QSt
<< SearchPluginIface::DescriptionInfo{tr("Path:"), path} \ << SearchPluginIface::DescriptionInfo{tr("Path:"), path} \
<< SearchPluginIface::DescriptionInfo{tr("Modified time:"), info.lastModified().toString("yyyy/MM/dd hh:mm:ss")}; << SearchPluginIface::DescriptionInfo{tr("Modified time:"), info.lastModified().toString("yyyy/MM/dd hh:mm:ss")};
ri.actionKey = path; ri.actionKey = path;
if (targetPhotographTypeMap[info.suffix()]) { if (FileIndexerConfig::getInstance()->ocrContentIndexTarget()[info.suffix()]) {
ri.type = 1;//1为ocr图片文件 ri.type = 1;//1为ocr图片文件
} else { } else {
ri.type = 0;//0为默认文本文件 ri.type = 0;//0为默认文本文件

View File

@ -32,7 +32,7 @@
using namespace UkuiSearch; using namespace UkuiSearch;
UkuiSearchService::UkuiSearchService(int &argc, char *argv[], const QString &applicationName) UkuiSearchService::UkuiSearchService(int &argc, char *argv[], const QString &applicationName)
: QtSingleApplication (applicationName, argc, argv) : QtSingleApplication (applicationName, argc, argv)
{ {
qDebug()<<"ukui search service constructor start"; qDebug()<<"ukui search service constructor start";
setApplicationVersion(QString("v%1").arg(VERSION)); setApplicationVersion(QString("v%1").arg(VERSION));
@ -82,6 +82,11 @@ void UkuiSearchService::parseCmd(const QString& msg, bool isPrimary)
QCommandLineOption statusOption(QStringList()<<"s"<<"status", tr("show status of file index service")); QCommandLineOption statusOption(QStringList()<<"s"<<"status", tr("show status of file index service"));
parser.addOption(statusOption); parser.addOption(statusOption);
QCommandLineOption forceUpdateOption(QStringList()<<"u"<<"update",
tr("stop current index job and perform incremental updates for<index type>"),
"indexType");
parser.addOption(forceUpdateOption);
if (isPrimary) { if (isPrimary) {
const QStringList args = QString(msg).split(' '); const QStringList args = QString(msg).split(' ');
parser.process(args); parser.process(args);
@ -94,6 +99,20 @@ void UkuiSearchService::parseCmd(const QString& msg, bool isPrimary)
qApp->quit(); qApp->quit();
return; return;
} }
if (parser.isSet(forceUpdateOption)) {
BatchIndexer::Targets targets = BatchIndexer::None;
QString target = parser.value(forceUpdateOption);
if(target == "all") {
targets = BatchIndexer::All;
} else if (target == "basic") {
targets = BatchIndexer::Basic;
} else if (target == "content") {
targets = BatchIndexer::Content;
} else if (target == "ocr") {
targets = BatchIndexer::Ocr;
}
m_indexScheduler->forceUpdate(targets);
}
} else { } else {
if (arguments().count() < 2) { if (arguments().count() < 2) {
parser.showHelp(); parser.showHelp();